training vi and normalizing flows wip\
This commit is contained in:
ritchie46 committed
1 parent
f9e66aa340
commit
eebd9b5e3d
7 files changed
+561
-517
No files matched your search
File diff suppressed because it is too large.
Load diff
@@ -14,7 +14,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 379,
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -39,7 +39,7 @@
|
||||
"# Draw samples from two Gaussian w.p. z_i ~ Bernoulli(phi)\n",
|
||||
"generative_m = np.array([stats.norm(2, 1), stats.norm(5, 1.8)])\n",
|
||||
"z = stats.bernoulli(0.75).rvs(100)\n",
|
||||
"x_i = np.array([g.rvs() for g in generative_m[z_i]])\n",
|
||||
"x_i = np.array([g.rvs() for g in generative_m[z]])\n",
|
||||
"\n",
|
||||
"# plot generated data and the latent distributions\n",
|
||||
"x = np.linspace(-5, 12, 150)\n",
|
||||
@@ -59,7 +59,7 @@
|
||||
"source": [
|
||||
"# Log likelihood\n",
|
||||
"Let $\\theta = \\{ \\mu, \\sigma \\}$.\n",
|
||||
"$$\\ell(\\theta) = \\sum_{i-1}^n \\log P(x_i; \\theta) $$\n",
|
||||
"$$\\ell(\\theta) = \\sum_{i=1}^n \\log P(x_i; \\theta) $$\n",
|
||||
"\n",
|
||||
"Or in vector form:\n",
|
||||
"\n",
|
||||
@@ -131,7 +131,7 @@
|
||||
"Jensen's inequality holds to equaltiy if and only if $X = E[X]$. This will be the case if we choose a distribution that leads to:\n",
|
||||
"\n",
|
||||
"$$ \\begin{eqnarray}\n",
|
||||
"\\frac{P(x|z) P(z)}{Q(z)} &=& \\frac{P(x, z)}{Q(z)} = 1\n",
|
||||
"\\frac{P(x|z) P(z)}{Q(z)} &=& \\frac{P(x, z)}{Q(z)} = c\n",
|
||||
"\\end{eqnarray}$$\n",
|
||||
"\n",
|
||||
"Wich means that \n",
|
||||
@@ -160,12 +160,12 @@
|
||||
"## Gaussian mixture model\n",
|
||||
"\n",
|
||||
"### Expectation\n",
|
||||
"Given a Gaussian mixture model, observed data points $x$ and current parameters $\\theta_t$ find $Q(z)$"
|
||||
"Given a Gaussian mixture model, observed data points $x$ and current parameters $\\theta_t$ find $Q(z; \\theta_t) = P(z|x; \\theta_t)$.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 377,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -193,7 +193,7 @@
|
||||
" 1.42972786, 2.82725842, 4.57300419, 2.59096887, 3.45567689])"
|
||||
]
|
||||
},
|
||||
"execution_count": 377,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@@ -205,7 +205,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 266,
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -217,7 +217,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 363,
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -235,7 +235,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 364,
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -244,7 +244,7 @@
|
||||
"<BarContainer object of 100 artists>"
|
||||
]
|
||||
},
|
||||
"execution_count": 364,
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
},
|
||||
@@ -263,7 +263,7 @@
|
||||
],
|
||||
"source": [
|
||||
"torch.manual_seed(1)\n",
|
||||
"\n",
|
||||
"g in better poste-rior a\n",
|
||||
"k = torch.tensor([2])\n",
|
||||
"mu = dist.Uniform(x_i.min(), x_i.max()).sample(k) \n",
|
||||
"std = torch.ones(k)\n",
|
||||
@@ -292,14 +292,15 @@
|
||||
"source": [
|
||||
"### Maximization\n",
|
||||
"\n",
|
||||
"Optimize $\\theta$ with the $Q(z)$ found in the expectation step.\n",
|
||||
"Optimize $\\theta$ with the $Q(z; \\theta_t)$ found in the expectation step.\n",
|
||||
"$$ \\text{ELBO} = \\sum_{z_i} Q(z_i) \\log ( \\frac{p(x_i| z_i; \\theta_{t + 1}) p(z_i)} {Q(z_i)}) $$\n",
|
||||
"\n",
|
||||
"$$ \\sum_{z_i} Q(z_i) \\log ( \\frac{p(x_i| z_i; \\theta) p(z_i)} {Q(z_i)}) $$"
|
||||
"$$ \\underset{\\theta_{t + 1}}{\\text{argmax ELBO}} $$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 370,
|
||||
"execution_count": 28,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -319,30 +320,21 @@
|
||||
"std = torch.ones(k)\n",
|
||||
"mu = torch.tensor(mu, dtype=torch.float, requires_grad=True)\n",
|
||||
"std = torch.tensor([1., 1.])\n",
|
||||
"std = torch.tensor(std, dtype=torch.float, requires_grad=True)\n",
|
||||
"phi = torch.rand(2, requires_grad=True)\n"
|
||||
"std = torch.tensor(std, dtype=torch.float, requires_grad=True)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 371,
|
||||
"execution_count": 31,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/opt/miniconda3/lib/python3.7/site-packages/ipykernel_launcher.py:4: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
|
||||
" after removing the cwd from sys.path.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"tensor(-512.5377, grad_fn=<AddBackward0>)"
|
||||
"tensor(-540.7363, grad_fn=<AddBackward0>)"
|
||||
]
|
||||
},
|
||||
"execution_count": 371,
|
||||
"execution_count": 31,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@@ -351,7 +343,7 @@
|
||||
"def get_elbo(mu, std, phi):\n",
|
||||
" elbo = 0\n",
|
||||
" for k in range(2):\n",
|
||||
" z_i = F.softmax(phi)[k]\n",
|
||||
" z_i = torch.tensor(1.) # prior on z\n",
|
||||
" mu_i = mu[k]\n",
|
||||
" std_i = std[k]\n",
|
||||
" \n",
|
||||
@@ -365,38 +357,29 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 372,
|
||||
"execution_count": 34,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/opt/miniconda3/lib/python3.7/site-packages/ipykernel_launcher.py:4: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
|
||||
" after removing the cwd from sys.path.\n",
|
||||
"/opt/miniconda3/lib/python3.7/site-packages/ipykernel_launcher.py:21: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"215.0631103515625 tensor([5.8801, 2.9457]) tensor([1.5491, 1.5054]) tensor([0.3965, 0.6035])\n",
|
||||
"214.99671936035156 tensor([5.6912, 2.8268]) tensor([1.5984, 1.4784]) tensor([0.4478, 0.5522])\n",
|
||||
"214.94085693359375 tensor([5.5913, 2.7706]) tensor([1.6360, 1.4637]) tensor([0.4746, 0.5254])\n",
|
||||
"214.90855407714844 tensor([5.5367, 2.7465]) tensor([1.6611, 1.4576]) tensor([0.4884, 0.5116])\n",
|
||||
"214.89120483398438 tensor([5.5059, 2.7380]) tensor([1.6776, 1.4564]) tensor([0.4954, 0.5046])\n",
|
||||
"214.88137817382812 tensor([5.4879, 2.7368]) tensor([1.6889, 1.4577]) tensor([0.4989, 0.5011])\n",
|
||||
"214.87564086914062 tensor([5.4766, 2.7388]) tensor([1.6970, 1.4600]) tensor([0.5006, 0.4994])\n",
|
||||
"214.8721160888672 tensor([5.4690, 2.7420]) tensor([1.7031, 1.4625]) tensor([0.5014, 0.4986])\n",
|
||||
"214.8697509765625 tensor([5.4635, 2.7455]) tensor([1.7079, 1.4651]) tensor([0.5018, 0.4982])\n",
|
||||
"214.8681182861328 tensor([5.4594, 2.7489]) tensor([1.7118, 1.4674]) tensor([0.5019, 0.4981])\n"
|
||||
"neg. ELBO: 147.22\tmu: tensor([2.8749, 5.8349])\tstd: tensor([1.4770, 1.5313])\n",
|
||||
"neg. ELBO: 146.09\tmu: tensor([2.7886, 5.6798])\tstd: tensor([1.4575, 1.5878])\n",
|
||||
"neg. ELBO: 145.73\tmu: tensor([2.7476, 5.5918])\tstd: tensor([1.4493, 1.6264])\n",
|
||||
"neg. ELBO: 145.62\tmu: tensor([2.7310, 5.5414])\tstd: tensor([1.4473, 1.6521])\n",
|
||||
"neg. ELBO: 145.59\tmu: tensor([2.7267, 5.5119])\tstd: tensor([1.4488, 1.6695])\n",
|
||||
"neg. ELBO: 145.57\tmu: tensor([2.7280, 5.4938])\tstd: tensor([1.4517, 1.6818])\n",
|
||||
"neg. ELBO: 145.57\tmu: tensor([2.7318, 5.4821])\tstd: tensor([1.4552, 1.6908])\n",
|
||||
"neg. ELBO: 145.56\tmu: tensor([2.7362, 5.4739])\tstd: tensor([1.4586, 1.6978])\n",
|
||||
"neg. ELBO: 145.56\tmu: tensor([2.7406, 5.4678])\tstd: tensor([1.4618, 1.7033])\n",
|
||||
"neg. ELBO: 145.56\tmu: tensor([2.7447, 5.4631])\tstd: tensor([1.4646, 1.7079])\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"lr = 0.01\n",
|
||||
"for i in range(10):\n",
|
||||
"for i in range(10):𝑝(𝑧𝑖)\n",
|
||||
" \n",
|
||||
" # expectation step\n",
|
||||
" with torch.no_grad():\n",
|
||||
@@ -410,26 +393,28 @@
|
||||
" with torch.no_grad():\n",
|
||||
" mu -= mu.grad * lr \n",
|
||||
" std -= std.grad * lr\n",
|
||||
" phi -= phi.grad * lr\n",
|
||||
" std.grad.zero_()\n",
|
||||
" mu.grad.zero_()\n",
|
||||
" phi.grad.zero_()\n",
|
||||
"\n",
|
||||
" print(negative_elbo.item(), mu.data, std.data, F.softmax(phi).data)"
|
||||
" \n",
|
||||
" print(\"neg. ELBO: {:.2f}\"\n",
|
||||
" \"\\tmu: {}\"\n",
|
||||
" \"\\tstd: {}\"\n",
|
||||
" \"\".format(negative_elbo.item(), mu.data, std.data))\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 384,
|
||||
"execution_count": 27,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[<matplotlib.lines.Line2D at 0x7f7ac0160470>]"
|
||||
"[<matplotlib.lines.Line2D at 0x7f7f3402bf28>]"
|
||||
]
|
||||
},
|
||||
"execution_count": 384,
|
||||
"execution_count": 27,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
},
|
||||
@@ -457,6 +442,98 @@
|
||||
"plt.plot(x, generative_m[1].pdf(x), color='black', lw=1, ls='-.')\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## GMM\n",
|
||||
"\n",
|
||||
"### Expectation step\n",
|
||||
"$$\\begin{eqnarray}\n",
|
||||
"w_{ij} := p(z_i = j|x_i; \\theta) \\\\\n",
|
||||
"\\mu_j := \\frac{\\sum_{i=1}^nw_{ij} x_i} {\\sum_{i=1}^nw_{ij}}\n",
|
||||
"\\end{eqnarray}$$\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"### Maximization step\n",
|
||||
"$$\\begin{eqnarray}\n",
|
||||
"\\mu_j := \\frac{\\sum_{i=1}^nw_{ij} x_i} {\\sum_{i=1}^nw_{ij}} \\\\\n",
|
||||
"\\sigma_j := \\sqrt{\\frac{\\sum_{i=1}^nw_{ij}(x_i - \\mu_j)^2} {\\sum_{i=1}^nw_{ij}}}\n",
|
||||
"\\end{eqnarray}$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 39,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[<matplotlib.lines.Line2D at 0x7f7f344ad438>]"
|
||||
]
|
||||
},
|
||||
"execution_count": 39,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"image/png": "iVBORw0KGgoAAAANSUhEUgAAA6IAAAFlCAYAAADxilWiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3iV9f3/8eednZCQxYYsAjJERghhKihi2QkqAk6sWq04UNviV2y12tqfrVqLLSqiIi5wYMABUpbIJmxQIBCSMEMgIWSvc//+OEBRGUnOfZJzktfjuriUc+77fb9QriTv81mGaZqIiIiIiIiI1BaPug4gIiIiIiIiDYsaUREREREREalVakRFRERERESkVqkRFRERERERkVqlRlRERERERERqlRpRERERERERqVVedfXgJk2amNHR0XX1eBEREREREXGiTZs2nTBNs+mF3quzRjQ6OpqUlJS6eryIiIiIiIg4kWEYGRd7T1NzRUREREREpFapERUREREREZFapUZUREREREREapUaUREREREREalVakRFRERERESkVqkRFRERERERkVqlRlRERERERERqlRpRERERERERqVVqREVERERERKRWqREVERERERGRWqVGVERERERERGqVGlERkQZuz7F8cgvL6jqGiIiINCBedR1ARETqxr7jBfz16x9Yvicbb0+DQR2acVNca67t2AxfL8+6jiciIiL1mBpREZEG5lRRGa8uSeX9dRkEeHvy+1914FRRGclbj/DfH7II9vdmRNeW3NY7kitbBdd1XBEREamH1IiKiDQQFZU2PliXwT+XpJJfUs74hEgeH3IFTQJ9AZgytCOr95/ki82H+GLzYT7bdIh5v+1Hl9ZqRkVERMRahmmadfLg+Ph4MyUlpU6eLSLSEP0xeSfvr8ugf7twnh7RmU4tG1/02uz8Ukb/exWeHgZfPTyAkACfWkwqIiIi9YFhGJtM04y/0HvarEhEpAFYuTeb99dlcHf/aD64p/clm1CApkG+vH57T46fLuXROVux2ermQ0sRERGpn9SIiojUc3nF5Uz5fDuxTRsxZWhHDMOo0n3dI0J4ZnRnvtubzb+Wpjo5pYiIiDQkakRFROq557/6gazTJbx8S3f8vKu3G+6tCZHc3LMN/1qayrLdWU5KKCIiIg2NGlERkXpsyQ9ZfLbpEA8Oakf3iJBq328YBn9J6kLnlo2ZPGcrmSeLnJBSREREGho1oiIi9VRuYRlPzttBxxZBPDK4fY3r+Hl78sbtPTEMg/s/2ERxWaWFKUVERKQhUiMqIlJP/WnBLvKKy3jllu74eDn25T4yPIBXx3Xnx6OneWf1AYsSioiISEOlRlREpB76evtRvtx2hEeua0/nVpfeIbeqru3YjEEdmvLOqgOUlGtUVERERGpOjaiISD1TUl7JMwt20rVNML8dFGtp7d8OjOVkYRmfphy0tK6IiIg0LGpERUTqmQVbj3CioIwnh3bEy9PaL/MJMWHERYbw5so0KiptltYWERGRhkONqIhIPWKaJu+sPkCH5kH0jQ23vL5hGPx2UDsO5Rbz1fajltcXERGRhkGNqIhIPbIuLYfdx/K5u380hmE45RmDOzbjiuaBvL5iP6ZpOuUZIiIiUr+pERURqUfeXX2A0ABvknq0dtozPDwMHhgYy56sfJbvOe6054iIiEj9pUZURKSeOJhTxH9/zGJCQiR+3p5Ofdaobq1oHeLP6yv2O/U5IiIiUj+pERURqSfeW5OOh2FwR98opz/L29OD+66OYWN6LhvTc5z+PBEREalf1IiKiNQDhaUVzE05yLAuLWgZ7F8rzxzXK5KwRj68oVFRERERqSY1oiIi9cDnmw+RX1LB3f1jau2Z/j6eTOwXzdLdx9l97HStPVdERETcnxpRERE3Z7OZzFqdTrc2wcRFhtTqs+/sG0WAj6dGRUVERKRa1IiKiLi571KzSTtRyN39Y5x2ZMvFhAT4cHPPNnyz8xj5JeW1+mwRERFxX2pERUTc3Lur02kW5Mvwq1rWyfMTu7eirMLG4l1ZdfJ8ERERcT9qREVE3Ni+4wWs3JvN7X2i8PGqmy/pcZGhtA7xZ8G2I3XyfBEREXE/akRFRNzYvM2H8PQwmJAQWWcZDMNgVLdWrNp3gpMFpXWWQ0RERNyHGlERETdlmibf7DhK37bhNA3yrdMso7u1otJm8s3OY3WaQ0RERNyDGlERETf149F80k8W1dna0PN1ahlEu2aBfLlV03NFRETk8tSIioi4qW92HMXDgF9d2byuo2AYBqO7tWJDeg5HThXXdRwRERFxcWpERUTc0NlpuX3ahhMeWLfTcs8a3a0VAF9t16ioiIiIXJoaURERN7QnK5+0E4UuMS33rOgmjejaJli754qIiMhlqREVEXFD32y3T8sd2qVFXUf5idHdWrHz8GnSsgvqOoqIiIi4MDWiIiJuxjRNvt5xlN4x4TRxkWm5Z43s2grDQKOiIiIicklqREVE3Ezq8QL2Zxcy/CrXGg0FaBHsR0J0GAu2HcE0zbqOIyIiIi5KjaiIiJv5evtRDAN+5WLTcs8a3b0VadmF7Dpyuq6jiIiIiItSIyoi4ma+2XGUhOgwmgX51XWUCxrepSVeHgZfanquiIiIXESVGlHDMIYahrHHMIx9hmE8eYnrehmGUWkYxs3WRRQRkbNSs/JJPV7gUrvl/lxoIx+ubt+EL7cdwWbT9FwRERH5pcs2ooZheAL/AYYBnYEJhmF0vsh1LwLfWh1SRETsvtlxDMOAYS46Lfes0d1bcSSvhC0Hc+s6ioiIiLigqoyIJgD7TNNMM02zDJgDJF7guoeBz4HjFuYTEZHzfLPjKL2iwmjW2DWn5Z51XcfmeHoYLNutbwkiIiLyS1VpRFsDB8/7/aEzr51jGEZrYAzwhnXRRETkfPuOF7AnK98ld8v9uWB/b3pGhrJ8d3ZdRxEREREXVJVG1LjAaz9f9PMqMMU0zcpLFjKM3xiGkWIYRkp2tn44ERGpjoU7jgIwtIvrrA/dsGED7733HgDjx4/nxx9/JCMjg8mTJ2Pb+RU7M4+TdbqkjlOKiIiIq6lKI3oIiDjv922An2+FGA/MMQwjHbgZmG4YRtLPC5mmOcM0zXjTNOObNm1aw8giIg3Tol3H6BkVSovgup+Wu2PHDpKSkrjpppvOvfbss88SFRWFj48PkZGRnD6wk6OzHuXtL5bUYVIRERFxRVVpRDcC7Q3DiDEMwwcYDyw4/wLTNGNM04w2TTMa+Ax40DTNZMvTiog0UMfzS9h15DSDOzWr0xz79u3jtttuY8iQIQwcOJDU1FTuuusuADp27EhAQAAtW7bk8ccf59uvkokaMpHnH7mT559/noqKijrNLiIiIq7jso2oaZoVwEPYd8P9EfjENM1dhmE8YBjGA84OKCIi8P3eEwBc077uZpPk5eUxcOBAOnXqRGpqKo899hh+fhcfnTUMg5tuvoWoe6axYsV3zJo1q/bCioiIiEvzqspFpml+A3zzs9cuuDGRaZoTHY8lIiLnW5maTZNAHzq3bFwnzz99+jTBwcHs3LmT0NDQKt93bcemzE0J5S9vfkxCTBhLliyhuLiYUaNGOTGtiIiIuLqqTM0VEZE6ZLOZfJ96gqvbN8XD40L7xznf2LFj2bx5c7WaUID+7Zrg5WGwct9JPD09ady4MWFhYU5KKSIiIu5CjaiIiIvbeSSPnMIyBl5R+9Ny8/PzKSoqIjk5mbi4uGrfH+TnTXx0KMvPnCeakJBA3759+fvf/05+fr7VcUVERMRNqBEVEXFxK/faj7sa0L5JrT63rKyMMWPG8Prrr+Pv71/jOoM6NGP3sXyO5dmPcfHw8CA1NZV77rkH0/z5aWAiIiLSEKgRFRFxcSv3nqBL68Y0CfSttWfabDYmTpxIUFAQkydPdqjWtR3sO/2u2HP83GuvvfYaaWlpvPrqqw7VFhEREfekRlRExIWdLilnc2Zure+W++qrr5Kens5HH32Ep6enQ7WuaB5Iy2A/VuzJPvean58fn3/+OS+++CIrV650NK6IiIi4GTWiIiIubM2+k1TYTK6pxfWhR44c4YUXXmDWrFkOTck9yzAMBnVoxqp9JyirsJ17PSoqitmzZzNhwgSOHDni8HNERETEfagRFRFxYStTs2nk40lcZPV2q3XEH/7wB+6//36uuOIKy2oO6tCUgtIKNmXk/uT1G264gYcfflijoiIiIg1Mlc4RFRGR2meaJiv3ZtOvXRN8vGrvc8OnnnqKqKgoS2v2b9cEb0+DFXuP0zc2/CfvPfnkkwDk5eURHBxs6XNFRETENWlEVETERR04Ucih3OJam5ZbXl7OM888Q2xsLI0aNbK0dqCvF72iw1ixO/uC7//www+MGjXK0meKiIiI61IjKiLior47c2zLwFraqKisrIxmzZrh4+PjlPrXdmjGnqx8jpwq/sV7nTt3ZsmSJU55roiIiLgeNaIiIi5q5d5sosMDiAwPcPqzjhw5wv79+5k0aRKGYTjlGYM62Bvq83fPPZ9hGAwbNoycnBynPF9ERERchxpREREXVFpRybq0nFqblvv444/zySefOPUZ7ZoF0jrEn+/2Hr/g+97e3kRHR/OXv/zFqTlERESk7qkRFRFxQSnpuRSXV9bK+aFLly5l/fr1PPXUU059jmEYDGjXhHVpOdhs5gWvefbZZ5k9ezZpaWlOzSIiIiJ1S42oiIgLWrk3G29P4xc7zFrNZrPx2GOP8corrxAQ4PwpwH1jw8krLueHo6cv+H7z5s159NFHnd4Ui4iISN1SIyoi4oK+25tNfFQYjXyde8rWwoUL8fT0JCkpyanPOetsY712/8mLXvP444/z/fffs379+lrJJCIiIrVPjaiIiIvJOl3C7mP5tbI+9MUXX+QPf/iD0zYo+rnmjf1o26QRa9Mu3og2atSI5557junTp9dKJhEREalLine truncated
|
||||
"text/plain": [
|
||||
"<Figure size 1152x432 with 1 Axes>"
|
||||
]
|
||||
},
|
||||
"metadata": {
|
||||
"needs_background": "light"
|
||||
},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"class EM:\n",
|
||||
" def __init__(self, k):\n",
|
||||
" self.k = k\n",
|
||||
" self.mu = None\n",
|
||||
" self.std = np.ones(k)\n",
|
||||
" self.w_ij = None\n",
|
||||
" self.phi = np.ones(k) / k\n",
|
||||
"\n",
|
||||
" def expectation_step(self, x):\n",
|
||||
" for z_i in range(self.k):\n",
|
||||
" self.w_ij[z_i] = stats.norm(self.mu[z_i], self.std[z_i]).pdf(x) * self.phi[z_i]\n",
|
||||
" \n",
|
||||
" # normalize zo that marginalizing z would lead to p = 1\n",
|
||||
" self.w_ij /= self.w_ij.sum(0)\n",
|
||||
"\n",
|
||||
" def maximization_step(self, x):\n",
|
||||
" self.phi = self.w_ij.mean(1)\n",
|
||||
" self.std = ((self.w_ij * (x - self.mu[:, None])**2).sum(1) / self.w_ij.sum(1))**0.5\n",
|
||||
" self.mu = (self.w_ij * x).sum(1) / self.w_ij.sum(1)\n",
|
||||
"\n",
|
||||
" def fit(self, x):\n",
|
||||
" self.mu = np.random.uniform(x.min(), x.max(), size=self.k)\n",
|
||||
" self.w_ij = np.zeros((self.k, x.shape[0]))\n",
|
||||
"\n",
|
||||
" last_mu = np.ones(self.k) * np.inf\n",
|
||||
" while ~np.all(np.isclose(self.mu, last_mu)):\n",
|
||||
" last_mu = self.mu\n",
|
||||
" self.expectation_step(x)\n",
|
||||
" self.maximization_step(x)\n",
|
||||
"\n",
|
||||
"m = EM(2)\n",
|
||||
"m.fit(np.array(x_i))\n",
|
||||
"\n",
|
||||
"fitted_m = [stats.norm(mu, std) for mu, std in zip(m.mu, m.std)]\n",
|
||||
"\n",
|
||||
"plt.figure(figsize=(16, 6))\n",
|
||||
"plt.vlines(x_i, 0, 0.01, color=np.array(['C0', 'C1'])[z])\n",
|
||||
"plt.plot(x, fitted_m[0].pdf(x))\n",
|
||||
"plt.plot(x, fitted_m[1].pdf(x))\n",
|
||||
"plt.plot(x, generative_m[0].pdf(x), color='black', lw=1, ls='-.')\n",
|
||||
"plt.plot(x, generative_m[1].pdf(x), color='black', lw=1, ls='-.')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 66 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 42 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 772 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 22 KiB |
@@ -0,0 +1,93 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Variational Autoencoders\n",
|
||||
"\n",
|
||||
"A Bayesian approach\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"## Suddenly a wild fashion-set appears...\n",
|
||||
"<img src=\"img/fashion-mnist-sprite.png\"/>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Generative model\n",
|
||||
"\n",
|
||||
"<img src=\"./img/plate-vae.png\"/>\n",
|
||||
"\n",
|
||||
"For each image $x_i$:\n",
|
||||
"\n",
|
||||
"$$\\begin{eqnarray}\n",
|
||||
"z_i &\\sim& P(z) \\\\\n",
|
||||
"x_i &\\sim& P(x|z_i)\n",
|
||||
"\\end{eqnarray}$$\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Bayesian inference\n",
|
||||
"\n",
|
||||
"$$P(z|x) = \\frac{P(x|z)P(z)}{P(x)} $$\n",
|
||||
"\n",
|
||||
"## Can we solve this with MCMC?\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Approximate distribution\n",
|
||||
"\n",
|
||||
"Let's define an approximate distribution $Q(Z) \\approx P(z|x)$. We optimize the variables of $Q(z)$ so that it will be 'closer' to the true posterior $P(z|x)$\n",
|
||||
"\n",
|
||||
"<img src=\"./img/easydist.png\"/>\n",
|
||||
"\n",
|
||||
"<img src=\"./img/dafuq.jpg\"/>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## KL-divergence\n",
|
||||
"There is a reason we choose to measure the 'closeseness' with the KL-divergence and not a real distance metric, such as the Wasserstein distance."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
Reference in new issue
Block a user