{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_openml\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import GPy\n",
    "from scipy import stats\n",
    "from scipy import optimize\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import metrics\n",
    "from tqdm import tqdm\n",
    "from functools import partial\n",
    "\n",
    "# http://krasserm.github.io/2018/03/21/bayesian-optimization/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Expected Improvement\n",
    "\n",
    "[derivation](http://ash-aldujaili.github.io/blog/2018/02/01/ei/)\n",
    "\n",
    "$$\\text{EI}(x) = \\mathbb{E}\\max(f(x^*) - f(x^+), 0)$$\n",
    "\n",
    "$$\\text{EI}(x) = (\\mu(x^*) - f(x^+)) \\Phi(\\frac{\\mu(x^*) - f(x^+)}{\\sigma(x^*)}) + \\sigma(x^*) \\phi(\\frac{\\mu(x^*) - f(x^+)}{\\sigma(x^*)})$$\n",
    "\n",
    "* $\\Phi$ = CDF standard normal\n",
    "* $\\phi$ = PDF standard normal\n",
    "* $x^*$ = proposed points\n",
    "* $x^+$ = current best\n",
    "\n",
    "Numerrically unstable if $\\sigma \\to 0$\n",
    "\n",
    "$$\\text{EI}(x) = (\\mu(x^*) - f(x^+)) \\Phi(Z) + \\sigma \\phi(Z)$$\n",
    "\n",
    "$$Z = \\begin{cases}\n",
    "    \\frac{\\mu(x^*) - f(x^+)}{\\sigma(x^*)},& \\text{if } \\sigma(x^*) > 0 \\\\\n",
    "    0,              & \\text{otherwise}\n",
    "\\end{cases}$$\n",
    "\n",
    "Let $\\mu(x^*) - f(x^+) := \\delta$.\n",
    "\n",
    "$$Z = \\begin{cases}\n",
    "    \\frac{\\delta}{\\sigma(x^*)},& \\text{if } \\sigma(x^*) > 0 \\\\\n",
    "    0,              & \\text{otherwise}\n",
    "\\end{cases}$$\n",
    "\n",
    "$$\\text{EI}(x) = (\\delta) \\Phi(Z) + \\sigma(x^*) \\phi(Z)$$\n",
    "\n",
    "Recommendation value for $\\xi = 0.01$. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def select_hyperparams_random(param_ranges):\n",
    "    \"\"\"\n",
    "    Select hyperparameters at random.\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    param_ranges : dict\n",
    "        Named parameter ranges.\n",
    "\n",
    "        Example:\n",
    "        \n",
    "        {\n",
    "            'foo': {\n",
    "                'range': [1, 10],\n",
    "                'type': 'float'\n",
    "            }\n",
    "            'bar': {\n",
    "                'range': [10, 1000],\n",
    "                'type': 'int'\n",
    "            }\n",
    "        }\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    selection : dict\n",
    "        Randomly selected hyperparameters within given boundaries.\n",
    "        \n",
    "        Example:\n",
    "        {'foo': 4.213, 'bar': 935}\n",
    "\n",
    "    \"\"\"\n",
    "    selection = {}\n",
    "    for k in param_ranges:\n",
    "        val = np.random.choice(\n",
    "            np.linspace(*param_ranges[k]['range'], num=100)\n",
    "        )\n",
    "\n",
    "        dtype = param_ranges[k]['type']\n",
    "        if dtype is 'int':\n",
    "            val = int(val)\n",
    "        selection[k] = val\n",
    "    return selection\n",
    "\n",
    "\n",
    "def expected_improvement(f, y_current, x_proposed):\n",
    "    \"\"\"\n",
    "    Return E(max(f_proposed - f_current), 0)\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "\n",
    "    f : GP predict function\n",
    "    y_current : float\n",
    "        Current best evaluation f(x+)\n",
    "    x_proposed : np.array\n",
    "        Proposal parameters. Shape: (1, 1)\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    expected_improvement : float\n",
    "        E(max(f_proposed - f_current), 0)\n",
    "    \"\"\"\n",
    "    mu, var = f(x_proposed)\n",
    "    std = var ** 0.5\n",
    "    delta = mu - y_current\n",
    "\n",
    "    # x / inf = 0\n",
    "    std[std == 0] = np.inf\n",
    "    z = delta / std\n",
    "    unit_norm = stats.norm()\n",
    "    return delta * unit_norm.cdf(z) + std * unit_norm.pdf(z)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'foo': 9.363636363636363, 'bar': 720}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_ranges = {\n",
    "    'foo': {\n",
    "        'range': [1, 10],\n",
    "        'type': 'float'\n",
    "    },\n",
    "    'bar': {\n",
    "        'range': [10, 1000],\n",
    "        'type': 'int'\n",
    "    }\n",
    "}\n",
    "\n",
    "select_hyperparams_random(param_ranges)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BayesOpt:\n",
    "    def __init__(\n",
    "        self, param_ranges, f, random_trials=5, optimization_trials=20, kernel=None\n",
    "    ):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "\n",
    "        param_ranges : dict\n",
    "        f : function\n",
    "            black box function to evaluate\n",
    "        random_trials : int\n",
    "            Number of random trials to run before optimization starts\n",
    "        optimization_trials : int\n",
    "            Number of optimization trials to run.\n",
    "            Together with the random_trials this is the total budget\n",
    "        kernel: GPy.kern.src.kern.Kern\n",
    "            GPy kernel for the Gaussian Process.\n",
    "            If None given, RBF kernel is used\n",
    "        \"\"\"\n",
    "        self.param_ranges = param_ranges\n",
    "        self.f = f\n",
    "        self.random_trials = random_trials\n",
    "        self.optimization_trials = optimization_trials\n",
    "        self.n_trials = random_trials + optimization_trials\n",
    "        self.x = np.zeros((self.n_trials, len(param_ranges)))\n",
    "        self.y = np.zeros((self.n_trials, 1))\n",
    "\n",
    "        if kernel is None:\n",
    "            self.kernel = GPy.kern.RBF(\n",
    "                input_dim=self.x.shape[1], variance=1, lengthscale=1\n",
    "            )\n",
    "        else:\n",
    "            self.kernel = kernel\n",
    "        self.gp = None\n",
    "        self.bounds = np.array([pr[\"range\"] for pr in param_ranges.values()])\n",
    "\n",
    "    @property\n",
    "    def best_params(self):\n",
    "        \"\"\"\n",
    "        Select best parameters.\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "        best_parameters : dict\n",
    "        \"\"\"\n",
    "        return self._prepare_kwargs(self.x[self.y.argmax()])\n",
    "\n",
    "    def fit(self):\n",
    "        self._random_search()\n",
    "        self._bayesian_search()\n",
    "\n",
    "    def _random_search(self):\n",
    "        \"\"\"\n",
    "        Run the random trials budget\n",
    "        \"\"\"\n",
    "        print(f\"Starting {self.random_trials} random trials...\")\n",
    "        for i in tqdm(range(self.random_trials)):\n",
    "            hp = select_hyperparams_random(self.param_ranges)\n",
    "            self.x[i] = np.array(list(hp.values()))\n",
    "            self.y[i] = self.f(hp)\n",
    "\n",
    "    def _bayesian_search(self):\n",
    "        \"\"\"\n",
    "        Run the Bayesian Optimization budget\n",
    "        \"\"\"\n",
    "        print(f\"Starting {self.optimization_trials} optimization trials...\")\n",
    "        for i in tqdm(\n",
    "            range(self.random_trials, self.random_trials + self.optimization_trials)\n",
    "        ):\n",
    "            self.x[i], self.y[i] = self._single_iter()\n",
    "\n",
    "    def _single_iter(self, x=None):\n",
    "        \"\"\"\n",
    "        Fit a GP and retrieve and evaluate a new\n",
    "        parameter proposal.\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "        out : tuple[np.array[flt], np.array[flt]]\n",
    "            (x, f(x))\n",
    "\n",
    "        \"\"\"\n",
    "        self._fit_gp()\n",
    "        if x is None:\n",
    "            x = self._new_proposal()\n",
    "        y = self.f(self._prepare_kwargs(x))\n",
    "        return x, y\n",
    "\n",
    "    def _fit_gp(self, noise_var=0):\n",
    "        \"\"\"\n",
    "        Fit a GP on the currently observed data points.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        noise_var : flt\n",
    "            GPY argmument noise_var\n",
    "        \"\"\"\n",
    "        mask = self.x.sum(axis=1) != 0\n",
    "        self.gp = GPy.models.GPRegression(\n",
    "            self.x[mask],\n",
    "            self.y[mask],\n",
    "            normalizer=True,\n",
    "            kernel=self.kernel,\n",
    "            noise_var=noise_var,\n",
    "        )\n",
    "        self.gp.optimize()\n",
    "\n",
    "    def _new_proposal(self, n=25):\n",
    "        \"\"\"\n",
    "        Get a new parameter proposal by maximizing\n",
    "        the acquisition function.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        n : int\n",
    "            Number of retries.\n",
    "            Each new retry the optimization is\n",
    "            started in another parameter location.\n",
    "            This improves the chance of finding a global optimum.\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "        proposal : dict\n",
    "            Example:\n",
    "           {'foo': 4.213, 'bar': 935}\n",
    "        \"\"\"\n",
    "\n",
    "        def f(x):\n",
    "            return -expected_improvement(\n",
    "                f=self.gp.predict, y_current=self.y.max(), x_proposed=x[None, :]\n",
    "            )\n",
    "\n",
    "        x0 = np.random.uniform(\n",
    "            low=self.bounds[:, 0], high=self.bounds[:, 1], size=(n, self.x.shape[1])\n",
    "        )\n",
    "        proposal = None\n",
    "        best_ei = np.inf\n",
    "        for x0_ in x0:\n",
    "            res = optimize.minimize(f, x0_, bounds=self.bounds)\n",
    "            if res.success and res.fun < best_ei:\n",
    "                best_ei = res.fun\n",
    "                proposal = res.x\n",
    "            if np.isnan(res.fun):\n",
    "                raise ValueError(\"NaN within bounds\")\n",
    "        return proposal\n",
    "\n",
    "    def _prepare_kwargs(self, x):\n",
    "        \"\"\"\n",
    "        Create a dictionary with named parameters\n",
    "        and the proper python types.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        x : np.array\n",
    "            Example:\n",
    "            [4.213, 935.03]\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "        hyperparameters : dict\n",
    "\n",
    "            Example:\n",
    "            {'foo': 4.213, 'bar': 935}\n",
    "        \"\"\"\n",
    "        # create hyper parameter dict\n",
    "        hp = dict(zip(self.param_ranges.keys(), x))\n",
    "        # cast values\n",
    "        for k in self.param_ranges:\n",
    "            if self.param_ranges[k][\"type\"] == \"int\":\n",
    "                hp[k] = int(hp[k])\n",
    "            elif self.param_ranges[k][\"type\"] == \"float\":\n",
    "                hp[k] = float(hp[k])\n",
    "            else:\n",
    "                raise ValueError(\"Parameter type not known\")\n",
    "        return hp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f11b2cc9240>]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 25524 characters]\n",
      "text/plain": [
       "<Figure size 1152x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = np.linspace(0, 10)\n",
    "\n",
    "def func(x):\n",
    "    return np.sin(2 * x) + (x / 3)**2 - x \n",
    "\n",
    "plt.figure(figsize=(16, 4))\n",
    "plt.title('A unknown function $f$')\n",
    "plt.xlabel('$x$')\n",
    "plt.ylabel('$f(x)$')\n",
    "plt.plot(x, func(x))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_params(hyperparams):\n",
    "    return func(**hyperparams)\n",
    "\n",
    "param_ranges = {\n",
    "    'x': {\n",
    "        'range': [0, 10],\n",
    "        'type': 'float'\n",
    "    }\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:00<00:00, 4742.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting 2 random trials...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " /opt/miniconda3/lib/python3.7/site-packages/paramz/transformations.py:111: RuntimeWarning:divide by zero encountered in log\n",
      " /opt/miniconda3/lib/python3.7/site-packages/paramz/transformations.py:108: RuntimeWarning:invalid value encountered in greater\n",
      " /opt/miniconda3/lib/python3.7/site-packages/GPy/kern/src/stationary.py:168: RuntimeWarning:overflow encountered in true_divide\n",
      " /opt/miniconda3/lib/python3.7/site-packages/GPy/kern/src/rbf.py:54: RuntimeWarning:invalid value encountered in multiply\n",
      " /opt/miniconda3/lib/python3.7/site-packages/paramz/transformations.py:113: RuntimeWarning:invalid value encountered in greater\n",
      " /opt/miniconda3/lib/python3.7/site-packages/GPy/kern/src/rbf.py:51: RuntimeWarning:overflow encountered in square\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 43480 characters]\n",
      "text/plain": [
       "<Figure size 1440x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 43988 characters]\n",
      "text/plain": [
       "<Figure size 1440x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 49524 characters]\n",
      "text/plain": [
       "<Figure size 1440x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 52180 characters]\n",
      "text/plain": [
       "<Figure size 1440x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 50948 characters]\n",
      "text/plain": [
       "<Figure size 1440x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 54100 characters]\n",
      "text/plain": [
       "<Figure size 1440x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 56872 characters]\n",
      "text/plain": [
       "<Figure size 1440x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 52068 characters]\n",
      "text/plain": [
       "<Figure size 1440x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "np.random.seed(3)\n",
    "bo = BayesOpt(param_ranges, evaluate_params, random_trials=2)\n",
    "bo._random_search()\n",
    "bo._fit_gp()\n",
    "new_proposal = bo._new_proposal()\n",
    "x_, y_ = bo._single_iter(new_proposal)\n",
    "\n",
    "for i in range(2, 10):\n",
    "    new_proposal = bo._new_proposal()\n",
    "    ei = expected_improvement(\n",
    "        f=bo.gp.predict, y_current=bo.y.max(), x_proposed=x[:, None]\n",
    "    )\n",
    "    x_, y_ = bo._single_iter(new_proposal)\n",
    "    bo.x[i], bo.y[i] = x_, y_\n",
    "\n",
    "    quants = bo.gp.predict_quantiles(x[:, None], quantiles=(10, 90))\n",
    "    mu, var = bo.gp.predict(x[:, None])\n",
    "\n",
    "    plt.figure(figsize=(20, 4))\n",
    "    plt.suptitle(f\"Iteration {i - 1}\")\n",
    "    plt.subplot(1, 2, 1)\n",
    "    plt.title(\"$f(x^*)$\")\n",
    "    plt.plot(x, mu, color=\"C0\", label=\"Mean\")\n",
    "    plt.fill_between(\n",
    "        x,\n",
    "        quants[0].flatten(),\n",
    "        quants[1].flatten(),\n",
    "        alpha=0.15,\n",
    "        edgecolor=\"C1\",\n",
    "        label=\"Confidence\",\n",
    "    )\n",
    "    plt.scatter(\n",
    "        bo.x[: bo.random_trials],\n",
    "        bo.y[: bo.random_trials],\n",
    "        color=\"r\",\n",
    "        label=\"random trials\",\n",
    "    )\n",
    "    plt.xlim(x.min() - 1, x.max() + 1)\n",
    "    mask = bo.x.sum(axis=1) != 0\n",
    "    plt.scatter(\n",
    "        bo.x[mask][bo.random_trials : -1],\n",
    "        bo.y[mask][bo.random_trials : -1],\n",
    "        color=\"g\",\n",
    "        label=\"BO trials\",\n",
    "    )\n",
    "\n",
    "    plt.legend()\n",
    "    plt.subplot(1, 2, 2)\n",
    "    plt.title(\"$EI(x)$\")\n",
    "    plt.plot(x, ei, color=\"C1\")\n",
    "    plt.vlines(new_proposal, 0, ei.max(), linestyle=\"--\", label=\"New proposal\")\n",
    "    plt.xlim(x.min() - 1, x.max() + 1)\n",
    "    plt.legend()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " /opt/miniconda3/lib/python3.7/site-packages/ipykernel_launcher.py:7: RuntimeWarning:divide by zero encountered in true_divide\n",
      " /opt/miniconda3/lib/python3.7/site-packages/ipykernel_launcher.py:7: RuntimeWarning:invalid value encountered in true_divide\n",
      " /opt/miniconda3/lib/python3.7/site-packages/scipy/stats/_distn_infrastructure.py:901: RuntimeWarning:invalid value encountered in greater\n",
      " /opt/miniconda3/lib/python3.7/site-packages/scipy/stats/_distn_infrastructure.py:901: RuntimeWarning:invalid value encountered in less\n",
      " /opt/miniconda3/lib/python3.7/site-packages/scipy/stats/_distn_infrastructure.py:1807: RuntimeWarning:invalid value encountered in greater_equal\n",
      " /opt/miniconda3/lib/python3.7/site-packages/scipy/stats/_distn_infrastructure.py:897: RuntimeWarning:invalid value encountered in greater_equal\n",
      " /opt/miniconda3/lib/python3.7/site-packages/scipy/stats/_distn_infrastructure.py:897: RuntimeWarning:invalid value encountered in less_equal\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, '$\\\\sigma(x^*)$')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 10024 characters]\n",
      "text/plain": [
       "<Figure size 504x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "delta = np.linspace(0, 1)\n",
    "sigma = np.linspace(0, 1)[::-1]\n",
    "# get a square grid w/ the delta and sigma inputs\n",
    "dd, ss = np.meshgrid(delta, sigma)\n",
    "\n",
    "# this is just the EI function\n",
    "z = dd / ss\n",
    "unit_norm = stats.norm()\n",
    "ei = dd * unit_norm.cdf(z) + ss * unit_norm.pdf(z)\n",
    "ei[np.isnan(ei)] = 0\n",
    "\n",
    "plt.figure(figsize=(7, 7))\n",
    "plt.title('$EI(x)$')\n",
    "plt.contourf(delta, sigma, ei)\n",
    "plt.xlabel('$\\delta$')\n",
    "plt.ylabel('$\\sigma(x^*)$')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.linspace(0, 1000)\n",
    "x = np.vstack([x, x, x, x]).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f11ae7272e8>]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 18272 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def n_dim_func(n):\n",
    "    a = np.random.uniform(-0.1, 0.1, (3, n))\n",
    "    \n",
    "    def f(x):\n",
    "        y = a @ x.T + 0.1\n",
    "        return np.sin(2 * y[0, :]) + np.abs(y[1, :])**0.5 + np.cos(y[2, :]**3)\n",
    "        \n",
    "    return f\n",
    "    \n",
    "    \n",
    "plt.plot(n_dim_func(x.shape[1])(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for n in range(10, 30, 2):\n",
    "    \n",
    "    func = n_dim_func(n)\n",
    "\n",
    "    def evaluate_params(hyperparams):\n",
    "        x = np.array(list(hyperparams.values()))[None, :]\n",
    "        return func(x)\n",
    "    \n",
    "    param_ranges = {k: {\n",
    "        'range': [0, 100],\n",
    "        'type': 'float'\n",
    "    } for k in range(n)}\n",
    "    \n",
    "    bo = BayesOpt(param_ranges, \n",
    "              evaluate_params, \n",
    "              random_trials=5, \n",
    "              optimization_trials=20,\n",
    "              kernel=GPy.kern.Matern32(input_dim=n) + GPy.kern.Matern52(input_dim=n, variance=5) + GPy.kern.RBF(input_dim=n))\n",
    "    bo.fit()\n",
    "    \n",
    "    bo_random = BayesOpt(param_ranges, \n",
    "              evaluate_params, \n",
    "              random_trials=25, \n",
    "              optimization_trials=0)\n",
    "    bo_random.fit()\n",
    "    \n",
    "    y_max = max(bo.y.max(), bo_random.y.max())\n",
    "    y_min = min(bo.y.min(), bo_random.y.min())\n",
    "    \n",
    "    plt.figure(figsize=(16, 4))\n",
    "    plt.xlabel('iteration')\n",
    "    plt.ylabel('$f(x)$')\n",
    "    plt.ylim(y_min * 0.98, y_max * 1.02)\n",
    "    plt.plot(np.arange(1, len(bo.y) + 1), np.maximum.accumulate(bo_random.y), '-o', label='random')\n",
    "    plt.plot(np.arange(1, len(bo.y) + 1), np.maximum.accumulate(bo.y), '-o', label='Bayesian optimization')\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
