{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Variational Autoencoders\n",
    "\n",
    "A *Neural Network* **and** a *Bayesian* approach\n",
    "\n",
    "<img src=\"img/fashion-mnist-sprite.png\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset and utilities\n",
    "\n",
    "Below we'll import some libraries and use some utility classes from `pytorch` and `torchvision`.\n",
    "\n",
    "<img src=\"img/auto-encoder.png\" width=\"900px\"> </img>\n",
    "<small>[1]</small>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchvision\n",
    "from torchvision import transforms\n",
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import DataLoader\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load data\n",
    "data_dir = './data'\n",
    "t = transforms.ToTensor()\n",
    "\n",
    "fashion = False\n",
    "if fashion:\n",
    "    dset = torchvision.datasets.FashionMNIST\n",
    "else:\n",
    "    dset = torchvision.datasets.MNIST\n",
    "\n",
    "ds_train = dset(\n",
    "    root=data_dir,\n",
    "    train=True,\n",
    "    transform=t,\n",
    "    download=True\n",
    ")\n",
    "\n",
    "ds_test = dset(\n",
    "    root=data_dir,\n",
    "    transform=t,\n",
    "    train=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[torch.Tensor, int]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Show types in dataset\n",
    "[type(a) for a in next(iter(ds_train))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fd66b698710>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 5096 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(next(iter(ds_train))[0].reshape(28, 28), cmap=\"Greys\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'img': (torch.float32, torch.Size([64, 1, 28, 28])),\n",
       " 'label': (torch.int64, torch.Size([64]))}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_size = 64\n",
    "\n",
    "# Create dataloaders utilities\n",
    "dl_train = DataLoader(ds_train, shuffle=True, batch_size=batch_size)\n",
    "dl_test = DataLoader(ds_test, batch_size=batch_size)\n",
    "\n",
    "# img and label\n",
    "{k: (v.dtype, v.shape) for (k, v) in zip(['img', 'label'], next(iter(dl_train)))}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_batch(model, criterion, x, opt=None):\n",
    "    reconstruction_x = model(x)\n",
    "    loss = criterion(reconstruction_x, x.view(-1, 784))\n",
    "\n",
    "    if opt is not None:\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "        opt.zero_grad()\n",
    "\n",
    "    return loss.item(), len(x)\n",
    "\n",
    "def fit(epochs, model, criterion, optim, dl_train, dl_test, device='cuda', log_between_epochs=True):\n",
    "    \"\"\"\n",
    "    :param epoch: int\n",
    "    :param model: torch.nn.Module\n",
    "    :param criterion: torch.optim.Optimizer\n",
    "    :param dl_train: torch.utils.data.DataLoader\n",
    "    :param dl_test: torch.utils.data.DataLoader\n",
    "    :param device: str\n",
    "        'cuda' or 'cpu'\n",
    "    \"\"\"\n",
    "    for epoch in range(epochs):\n",
    "        \n",
    "        # train\n",
    "        model.train()\n",
    "        for i, (x, y) in enumerate(dl_train):\n",
    "            loss, n = loss_batch(model, criterion, x.to(device), optim)\n",
    "            \n",
    "            if i % 150 == 0 and log_between_epochs:\n",
    "                print(\"Loss: {:6.5f},\\t iteration: {:4d}\".format(loss / n, i))\n",
    "        \n",
    "        # evaluate test set\n",
    "        model.eval()\n",
    "        with torch.no_grad():\n",
    "            losses, nums = zip(\n",
    "                *[loss_batch(model, criterion, x.to(device)) for x, y in dl_test]\n",
    "            )\n",
    "        val_loss = np.sum(np.multiply(losses, nums)) / np.sum(nums)\n",
    "        print(\"\\nEpoch: {:4d},\\t Test loss: {:6.5f}\".format(epoch, val_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class AutoEncoder(nn.Module):\n",
    "    def __init__(self, input_size=784, z_size=20):\n",
    "        super().__init__()\n",
    "        # Implement autoencoder\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = x.view(-1, 784)\n",
    "        z = self.encoder(x)\n",
    "        x = self.decoder(z)\n",
    "        \n",
    "        if self.training:\n",
    "            return x\n",
    "        else:\n",
    "            return F.sigmoid(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class AutoEncoder(nn.Module):\n",
    "    def __init__(self, input_size=784, z_size=20):\n",
    "        super().__init__()\n",
    "        hidden_size = int((input_size - z_size) / 2 + z_size)\n",
    "        self.encoder = nn.Sequential(\n",
    "            nn.Linear(input_size, hidden_size),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(hidden_size, z_size)\n",
    "        )\n",
    "        self.decoder = nn.Sequential(\n",
    "            nn.Linear(z_size, hidden_size),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(hidden_size, input_size),\n",
    "        )\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = x.view(-1, 784)\n",
    "        z = self.encoder(x)\n",
    "        x = self.decoder(z)\n",
    "        \n",
    "        if self.training:\n",
    "            return x\n",
    "        else:\n",
    "            return F.sigmoid(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "On cuda!\n"
     ]
    }
   ],
   "source": [
    "model = AutoEncoder()\n",
    "optim = torch.optim.Adam(model.parameters(), lr=0.01)\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "epochs = 1\n",
    "device = 'cpu'\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    print('On cuda!')\n",
    "    model.cuda()\n",
    "    device = 'cuda'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 0.01081,\t iteration:    0\n",
      "Loss: 0.00195,\t iteration:  150\n",
      "Loss: 0.00163,\t iteration:  300\n",
      "Loss: 0.00167,\t iteration:  450\n",
      "Loss: 0.00165,\t iteration:  600\n",
      "Loss: 0.00162,\t iteration:  750\n",
      "Loss: 0.00165,\t iteration:  900\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/miniconda3/lib/python3.7/site-packages/torch/nn/functional.py:1351: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\n",
      "  warnings.warn(\"nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch:    0,\t Test loss: 0.67246\n"
     ]
    }
   ],
   "source": [
    "fit(epochs, model, criterion, optim, dl_train, dl_test, device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 5264 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fd67021bc50>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 4720 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    model.eval()\n",
    "    for x, _ in ds_test:\n",
    "        x_reconstruction = model(x.to(device)).cpu()\n",
    "        break\n",
    "\n",
    "plt.imshow(x_reconstruction.view(28, 28), cmap='Greys')\n",
    "plt.show()\n",
    "plt.imshow(x.view(28, 28), cmap='Greys')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Variational Autoencoder\n",
    "\n",
    "## *A neural networks perspective*\n",
    "\n",
    "Let $Q(z|x)$ be the encoder, and $P(x|z)$ the decoder. Both are neural networks.\n",
    "\n",
    "We want to be able to generate new data, so we modify the autoencoder so that it has a random process at it's core.\n",
    "\n",
    "\n",
    "$$ \\begin{eqnarray}\n",
    "Q_{\\theta}(\\cdot) &=& \\{ \\mu_{\\theta}, \\log \\sigma^2_{\\theta} \\}\\\\\n",
    "z &\\sim& \\mathcal{N}(\\mu_{\\theta}, \\sigma^2_{\\theta})\n",
    "\\end{eqnarray}$$\n",
    "\n",
    "Now it's of course very easy for $E(\\cdot)$ to cheat and become an autoencoder by just learning $E_{\\theta}(\\cdot) = \\{ \\mu_{\\theta}, 0 \\}$. Without variance the random process isn't quite random.\n",
    "\n",
    "Therefore, we need a regularization term, the Kullback-Leibler-Divergence. \n",
    "\n",
    "$$ D_{KL}(Q(z|x) || P(z)) $$\n",
    "\n",
    "Where we don't want the $z$ values we draw from the stochastic decoder to deviate too much from a unit Gaussian.\n",
    "\n",
    "$$ \\begin{eqnarray}\n",
    "z &\\sim& P(z) \\\\ \n",
    "  &\\sim& \\mathcal{N}(0, \\text{diag}(\\sigma^2)) \n",
    "\\end{eqnarray}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generative model\n",
    "## *A Bayesian perspective*\n",
    "\n",
    "<img src=\"./img/plate-vae.png\"/>\n",
    "\n",
    "For each image $x_i$:\n",
    "\n",
    "$$\\begin{eqnarray}\n",
    "z_i &\\sim& \\mathcal{N}(\\mu_i, \\sigma_i^2) \\\\\n",
    "x_i &\\sim& P(x_i|z_i)\n",
    "\\end{eqnarray}$$\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bayesian inference\n",
    "\n",
    "$$P(z|x) = \\frac{P(x|z)P(z)}{P(x)} $$\n",
    "\n",
    "## Can we solve this with MCMC?\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Approximate distribution\n",
    "\n",
    "Let's define an approximate distribution $Q(Z) \\approx P(z|x)$. We optimize the variables of $Q(z)$ so that it will be 'closer' to the true posterior $P(z|x)$\n",
    "\n",
    "<img src=\"./img/easydist.png\"/>\n",
    "\n",
    "<img src=\"./img/dafuq.jpg\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## KL-divergence\n",
    "There is a reason we choose to measure the 'closeseness' with the KL-divergence and not a real distance metric, such as the Wasserstein distance.\n",
    "\n",
    "$$\\begin{eqnarray}\n",
    "D_{\\text{KL}}(Q(z) \\: || \\: P(z|x)) &=& \\int_{z} Q(z) \\log \\frac{Q(z)}{P(z|x)}\\text{dz} \\\\\n",
    "&=& \\int_{z} q(z) \\log \\frac{q(z)p(x)}{p(z, x)}\\text{d}z  &\\qquad \\tiny{\\text{rewrite the posterior as}\\frac{p(z, x)}{p(x)} } \\\\\n",
    "&=& \\int_{z} Q(z) \\log \\frac{Q(z)}{P(z, x)}\\text{d}z + \\int_{z} Q(z) \\log P(x) \\text{d} z \\\\\n",
    "&=&  \\int_{z} Q(z) \\log \\frac{Q(z)}{P(z, x)}\\text{d}z +  \\log P(x) &\\qquad \\tiny{P(x)\\text{ is not parameterized by }z \\text{and} \\int_{z} Q(z) \\text{d} z = 1} \\\\\n",
    "&=& \\log P(x) -\\int_{z} Q(z) \\log \\frac{P(z, x)}{Q(z)}\\text{d}z &\\qquad \\tiny{\\log A = - \\log \\frac{q}{A}} \\\\\n",
    "&=& \\log P(x) - \\underbrace{ E_{z \\sim Q}[\\log \\frac{P(z, x)}{Q(z)} }_{\\text{ELBO}} ]\n",
    "\\end{eqnarray}\n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reconstruction loss and KL-divergence\n",
    "In the derivation of Variational Autoencoders, the ELBO is often written in two terms; a reconstruction error and a KL-term on the variational distribution $Q_z$ and the prior $P(z)$\n",
    "\n",
    "First we rewrite the joint probability $P(z, x)$ into conditional probability $P(x|z)P(z)$.\n",
    "\n",
    "$$ \\text{ELBO}  = E_{z \\sim Q}[\\log \\frac{P(x|z)P(z)}{Q(z)} ] $$\n",
    "\n",
    "Then we expand the equation and thereby isolating the reconstruction error $\\log P(x|z)$, i.e. the log likelihood.\n",
    "\n",
    "$$ \\text{ELBO}  = E_{z \\sim Q}[\\log P(x|z)] +  E_{z \\sim Q}[\\log \\frac{P(z)}{Q(z)}]$$\n",
    "\n",
    "If we rewrite the $E_{z \\sim Q}[\\log \\frac{P(z)}{Q(z)}]$ in the integral form $\\int_{z} Q(z)\\log\\frac{P(z)}{Q(z)}dz$, we can observe that this is the KL-divergence between the prior $P(z)$ and the variational distribution $Q(z)$. Resulting in an ELBO defined by the reconstruction error and $-D_{KL}(Q(z)||P(z)).$\n",
    "\n",
    "$$ \\text{ELBO}  = E_{z \\sim Q}[\\log P(x|z)] - D_{KL}(Q(z)||P(z))$$\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VAE(nn.Module):\n",
    "    def __init__(self, input_size=784, z_size=20):\n",
    "        super().__init__()\n",
    "        hidden_size = int((input_size - z_size) / 2 + z_size)\n",
    "        self.z_size = z_size\n",
    "        self.encoder = nn.Sequential(\n",
    "            nn.Linear(input_size, hidden_size),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(hidden_size, hidden_size),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(hidden_size, z_size * 2)\n",
    "        )\n",
    "        self.decoder = nn.Sequential(\n",
    "            nn.Linear(z_size, hidden_size),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(hidden_size, input_size),\n",
    "            nn.Sigmoid()\n",
    "        )\n",
    "        \n",
    "    def reparameterize(self, mu, log_var):\n",
    "        std = torch.exp(0.5 * log_var)\n",
    "        eps = torch.randn_like(std)  # unit gaussian\n",
    "        z = mu + eps * std \n",
    "        return z\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.view(-1, 784)\n",
    "        variational_params = self.encoder(x)\n",
    "        mu = variational_params[..., :self.z_size]\n",
    "        log_var = variational_params[..., self.z_size:]\n",
    "        z = self.reparameterize(mu, log_var)\n",
    "        return self.decoder(z), z, mu, log_var\n",
    "\n",
    "\n",
    "def loss_batch(model, criterion, x, opt=None):\n",
    "    reconstruction_x, z, mu, log_var = model(x)\n",
    "    loss = criterion(reconstruction_x, x.view(-1, 784))\n",
    "    \n",
    "    # see Appendix B from VAE paper:\n",
    "    # Kingma and Welling. Auto-Encoding Variational Bayes. ICLR, 2014\n",
    "    # https://arxiv.org/abs/1312.6114\n",
    "    # 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)\n",
    "    kl = -0.5 * torch.sum(1 + log_var - mu.pow(2) - log_var.exp())\n",
    "    \n",
    "    loss += kl\n",
    "\n",
    "    if opt is not None:\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "        opt.zero_grad()\n",
    "\n",
    "    return loss.item(), len(x)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "On cuda!\n"
     ]
    }
   ],
   "source": [
    "model = VAE()\n",
    "optim = torch.optim.Adam(model.parameters(), lr=0.01)\n",
    "criterion = nn.BCELoss(reduction='sum')\n",
    "epochs = 25\n",
    "device = 'cpu'\n",
    "batch_size = 256\n",
    "\n",
    "dl_train = DataLoader(ds_train, shuffle=True, batch_size=batch_size)\n",
    "dl_test = DataLoader(ds_test, batch_size=batch_size)\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    print('On cuda!')\n",
    "    model.cuda()\n",
    "    device = 'cuda'\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch:    0,\t Test loss: 45651.31713\n",
      "\n",
      "Epoch:    1,\t Test loss: 44757.27022\n",
      "\n",
      "Epoch:    2,\t Test loss: 43418.09588\n",
      "\n",
      "Epoch:    3,\t Test loss: 43290.16544\n",
      "\n",
      "Epoch:    4,\t Test loss: 43172.56996\n",
      "\n",
      "Epoch:    5,\t Test loss: 43022.13056\n",
      "\n",
      "Epoch:    6,\t Test loss: 43347.33919\n",
      "\n",
      "Epoch:    7,\t Test loss: 42884.59440\n",
      "\n",
      "Epoch:    8,\t Test loss: 42916.98715\n",
      "\n",
      "Epoch:    9,\t Test loss: 42681.94180\n",
      "\n",
      "Epoch:   10,\t Test loss: 42751.86753\n",
      "\n",
      "Epoch:   11,\t Test loss: 42673.31136\n",
      "\n",
      "Epoch:   12,\t Test loss: 42635.39283\n",
      "\n",
      "Epoch:   13,\t Test loss: 42672.09193\n",
      "\n",
      "Epoch:   14,\t Test loss: 42560.42911\n",
      "\n",
      "Epoch:   15,\t Test loss: 42571.33200\n",
      "\n",
      "Epoch:   16,\t Test loss: 42495.00136\n",
      "\n",
      "Epoch:   17,\t Test loss: 42640.09606\n",
      "\n",
      "Epoch:   18,\t Test loss: 42661.70678\n",
      "\n",
      "Epoch:   19,\t Test loss: 42861.99558\n",
      "\n",
      "Epoch:   20,\t Test loss: 42472.55930\n",
      "\n",
      "Epoch:   21,\t Test loss: 42630.67527\n",
      "\n",
      "Epoch:   22,\t Test loss: 42511.97979\n",
      "\n",
      "Epoch:   23,\t Test loss: 42701.39958\n",
      "\n",
      "Epoch:   24,\t Test loss: 42459.78887\n"
     ]
    }
   ],
   "source": [
    "fit(epochs, model, criterion, optim, dl_train, dl_test, device, log_between_epochs=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate new samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 5892 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    z = torch.randn(20)\n",
    "    x = model.decoder(z)\n",
    "    plt.imshow(x.view(28, 28), Cmap='Greys')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# References\n",
    "[1] https://lilianweng.github.io/lil-log/2018/08/12/from-autoencoder-to-beta-vae.html <br>\n",
    "[2] https://jaan.io/what-is-variational-autoencoder-vae-tutorial/ <br>\n",
    "[3] https://www.ritchievink.com/blog/2019/06/10/bayesian-inference-how-we-are-able-to-chase-the-posterior/ <br>\n",
    "[4] https://www.ritchievink.com/blog/2019/09/16/variational-inference-from-scratch/ <br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
