From 5c62d322f949175d17fcf0c39d0d5ce5b1af9b78 Mon Sep 17 00:00:00 2001 From: Jonathan Taylor Date: Sun, 6 Aug 2023 11:18:48 -0700 Subject: [PATCH] deep learning lab run --- Ch10-deeplearning-lab.ipynb | 1480 ++++++++++++++++++++--------------- README.md | 3 + requirements.txt | 2 +- 3 files changed, 862 insertions(+), 623 deletions(-) diff --git a/Ch10-deeplearning-lab.ipynb b/Ch10-deeplearning-lab.ipynb index 4f408b5..c7e788c 100644 --- a/Ch10-deeplearning-lab.ipynb +++ b/Ch10-deeplearning-lab.ipynb @@ -2,9 +2,10 @@ "cells": [ { "cell_type": "markdown", - "id": "fd22c2e6", + "id": "b672dcf6", "metadata": {}, "source": [ + "\n", "# Chapter 10\n", "\n", "# Lab: Deep Learning\n", @@ -17,14 +18,16 @@ "[pytorch.org/tutorials](https://pytorch.org/tutorials/beginner/basics/intro.html).\n", "Much of our code is adapted from there, as well as the `pytorch_lightning` documentation. {The precise URLs at the time of writing are and .}\n", "\n", - "We start with several standard imports that we have seen in other contexts." + "We start with several standard imports that we have seen before." ] }, { "cell_type": "code", "execution_count": 1, - "id": "aeb913e2", - "metadata": {}, + "id": "9b5b8319", + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [], "source": [ "import numpy as np, pandas as pd\n", @@ -45,7 +48,7 @@ }, { "cell_type": "markdown", - "id": "36a4ac8d", + "id": "c0cb2c87", "metadata": {}, "source": [ "### Torch-Specific Imports\n", @@ -58,19 +61,19 @@ { "cell_type": "code", "execution_count": 2, - "id": "3a3e91e7", + "id": "f66f362c", "metadata": {}, "outputs": [], "source": [ "import torch\n", "from torch import nn\n", "from torch.optim import RMSprop\n", - "from torch.utils.data import TensorDataset" + "from torch.utils.data import TensorDataset\n" ] }, { "cell_type": "markdown", - "id": "6e89c676", + "id": "5ce27a8d", "metadata": {}, "source": [ "There are several other helper packages for `torch`. For instance,\n", @@ -84,18 +87,18 @@ { "cell_type": "code", "execution_count": 3, - "id": "3f4fdb8c", + "id": "17bce9cb", "metadata": {}, "outputs": [], "source": [ "from torchmetrics import (MeanAbsoluteError,\n", " R2Score)\n", - "from torchinfo import summary" + "from torchinfo import summary\n" ] }, { "cell_type": "markdown", - "id": "b7b12e5c", + "id": "a14990c3", "metadata": {}, "source": [ "The package `pytorch_lightning` is a somewhat higher-level\n", @@ -108,17 +111,17 @@ { "cell_type": "code", "execution_count": 4, - "id": "71cc6f03", + "id": "4c3b6e43", "metadata": {}, "outputs": [], "source": [ "from pytorch_lightning import Trainer\n", - "from pytorch_lightning.loggers import CSVLogger" + "from pytorch_lightning.loggers import CSVLogger\n" ] }, { "cell_type": "markdown", - "id": "8394fc0a", + "id": "e5bc78bd", "metadata": {}, "source": [ "In order to reproduce results we use `seed_everything()`. We will also instruct `torch` to use deterministic algorithms\n", @@ -128,7 +131,7 @@ { "cell_type": "code", "execution_count": 5, - "id": "fd1290f1", + "id": "6839d8ad", "metadata": {}, "outputs": [ { @@ -142,12 +145,12 @@ "source": [ "from pytorch_lightning import seed_everything\n", "seed_everything(0, workers=True)\n", - "torch.use_deterministic_algorithms(True, warn_only=True)" + "torch.use_deterministic_algorithms(True, warn_only=True)\n" ] }, { "cell_type": "markdown", - "id": "f29a6c10", + "id": "1f1490e7", "metadata": {}, "source": [ "We will use several datasets shipped with `torchvision` for our\n", @@ -158,8 +161,10 @@ { "cell_type": "code", "execution_count": 6, - "id": "c85308d1", - "metadata": {}, + "id": "07dba2dd", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [], "source": [ "from torchvision.io import read_image\n", @@ -174,7 +179,7 @@ }, { "cell_type": "markdown", - "id": "6ca24f46", + "id": "36ec305c", "metadata": {}, "source": [ "We have provided a few utilities in `ISLP` specifically for this lab.\n", @@ -192,19 +197,19 @@ { "cell_type": "code", "execution_count": 7, - "id": "4de9f03c", + "id": "89763447", "metadata": {}, "outputs": [], "source": [ "from ISLP.torch import (SimpleDataModule,\n", " SimpleModule,\n", " ErrorTracker,\n", - " rec_num_workers)" + " rec_num_workers)\n" ] }, { "cell_type": "markdown", - "id": "d81a775a", + "id": "dfc13283", "metadata": {}, "source": [ "In addition we have included some helper\n", @@ -221,19 +226,19 @@ { "cell_type": "code", "execution_count": 8, - "id": "4ef95fe3", + "id": "ac5ab1b0", "metadata": {}, "outputs": [], "source": [ "from ISLP.torch.imdb import (load_lookup,\n", " load_tensor,\n", " load_sparse,\n", - " load_sequential)" + " load_sequential)\n" ] }, { "cell_type": "markdown", - "id": "d8d760ad", + "id": "04f757bd", "metadata": {}, "source": [ "Finally, we introduce some utility imports not directly related to\n", @@ -250,17 +255,19 @@ { "cell_type": "code", "execution_count": 9, - "id": "493821a9", - "metadata": {}, + "id": "5a5468e6", + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [], "source": [ "from glob import glob\n", - "import json" + "import json\n" ] }, { "cell_type": "markdown", - "id": "c415e4e5", + "id": "690a5255", "metadata": {}, "source": [ "## Single Layer Network on Hitters Data\n", @@ -270,17 +277,19 @@ { "cell_type": "code", "execution_count": 10, - "id": "a427e224", - "metadata": {}, + "id": "d635398b", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [], "source": [ "Hitters = load_data('Hitters').dropna()\n", - "n = Hitters.shape[0]" + "n = Hitters.shape[0]\n" ] }, { "cell_type": "markdown", - "id": "f1ca2542", + "id": "a0fd86f3", "metadata": {}, "source": [ " We will fit two linear models (least squares and lasso) and compare their performance\n", @@ -296,18 +305,20 @@ { "cell_type": "code", "execution_count": 11, - "id": "feac4b15", - "metadata": {}, + "id": "2c621749", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [], "source": [ "model = MS(Hitters.columns.drop('Salary'), intercept=False)\n", "X = model.fit_transform(Hitters).to_numpy()\n", - "Y = Hitters['Salary'].to_numpy()" + "Y = Hitters['Salary'].to_numpy()\n" ] }, { "cell_type": "markdown", - "id": "ad0d90b5", + "id": "f32a66d7", "metadata": {}, "source": [ "The `to_numpy()` method above converts `pandas`\n", @@ -320,7 +331,7 @@ }, { "cell_type": "markdown", - "id": "35acacc2", + "id": "b5565350", "metadata": {}, "source": [ "We now split the data into test and training, fixing the random\n", @@ -330,7 +341,7 @@ { "cell_type": "code", "execution_count": 12, - "id": "6a75a640", + "id": "cd19596a", "metadata": {}, "outputs": [], "source": [ @@ -345,7 +356,7 @@ }, { "cell_type": "markdown", - "id": "edc24bfb", + "id": "b82951aa", "metadata": {}, "source": [ "### Linear Models\n", @@ -355,7 +366,7 @@ { "cell_type": "code", "execution_count": 13, - "id": "32d8de64", + "id": "e4bfcd5a", "metadata": {}, "outputs": [ { @@ -377,7 +388,7 @@ }, { "cell_type": "markdown", - "id": "d60340ef", + "id": "bf3523a8", "metadata": {}, "source": [ "Next we fit the lasso using `sklearn`. We are using\n", @@ -385,13 +396,13 @@ "The specialized solver we used in Section 6.5.2 uses only mean squared error. So here, with a bit more work, we create a cross-validation grid and perform the cross-validation directly. \n", "\n", "We encode a pipeline with two steps: we first normalize the features using a `StandardScaler()` transform,\n", - "and then fit the lasso without further normalization." + "and then fit the lasso without further normalization. " ] }, { "cell_type": "code", "execution_count": 14, - "id": "46786db6", + "id": "36f8290b", "metadata": {}, "outputs": [], "source": [ @@ -403,7 +414,7 @@ }, { "cell_type": "markdown", - "id": "e7ff854a", + "id": "9f1bf0ad", "metadata": {}, "source": [ "We need to create a grid of values for $\\lambda$. As is common practice, \n", @@ -414,8 +425,10 @@ { "cell_type": "code", "execution_count": 15, - "id": "2b55b38b", - "metadata": {}, + "id": "5015c394", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [], "source": [ "X_s = scaler.fit_transform(X_train)\n", @@ -427,7 +440,7 @@ }, { "cell_type": "markdown", - "id": "977dea00", + "id": "89f6b4e4", "metadata": {}, "source": [ "Note that we had to transform the data first, since the scale of the variables impacts the choice of $\\lambda$.\n", @@ -437,7 +450,7 @@ { "cell_type": "code", "execution_count": 16, - "id": "cc44c559", + "id": "b5dc7d68", "metadata": {}, "outputs": [], "source": [ @@ -453,7 +466,7 @@ }, { "cell_type": "markdown", - "id": "fca88ebb", + "id": "0b64a882", "metadata": {}, "source": [ "We extract the lasso model with best cross-validated mean absolute error, and evaluate its\n", @@ -464,8 +477,10 @@ { "cell_type": "code", "execution_count": 17, - "id": "1bd1fd13", - "metadata": {}, + "id": "2c49196f", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [ { "data": { @@ -486,7 +501,7 @@ }, { "cell_type": "markdown", - "id": "f19cd1f2", + "id": "61f016c9", "metadata": {}, "source": [ "This is similar to the results we got for the linear model fit by least squares. However, these results can vary a lot for different train/test splits; we encourage the reader to try a different seed in code block 12 and rerun the subsequent code up to this point.\n", @@ -498,13 +513,13 @@ "Typically this is done in `pytorch` by sub-classing a generic\n", "representation of a network, which is the approach we take here.\n", "Although this example is simple, we will go through the steps in some detail, since it will serve us well\n", - "for the more complex examples to follow." + "for the more complex examples to follow.\n" ] }, { "cell_type": "code", "execution_count": 18, - "id": "4b2ccc5a", + "id": "df5864e2", "metadata": {}, "outputs": [], "source": [ @@ -521,12 +536,12 @@ "\n", " def forward(self, x):\n", " x = self.flatten(x)\n", - " return torch.flatten(self.sequential(x))" + " return torch.flatten(self.sequential(x))\n" ] }, { "cell_type": "markdown", - "id": "5da217b2", + "id": "4b05cc28", "metadata": {}, "source": [ "The `class` statement identifies the code chunk as a\n", @@ -562,16 +577,16 @@ { "cell_type": "code", "execution_count": 19, - "id": "108ad1f1", + "id": "da601fe1", "metadata": {}, "outputs": [], "source": [ - "hit_model = HittersModel(X.shape[1])" + "hit_model = HittersModel(X.shape[1])\n" ] }, { "cell_type": "markdown", - "id": "46d750dd", + "id": "326f3b54", "metadata": {}, "source": [ "The object `self.sequential` is a composition of four maps. The\n", @@ -579,24 +594,28 @@ "for the weights and *intercept* of the map (often called the *bias*). This layer\n", "is then mapped to a ReLU layer followed by a 40% dropout layer, and finally a\n", "linear map down to 1 dimension, again with a bias. The total number of\n", - "trainable parameters is therefore $50\\times 19+50+50+1=1051$." + "trainable parameters is therefore $50\\times 19+50+50+1=1051$.\n", + "\n", + " " ] }, { "cell_type": "markdown", - "id": "e0c250c5", + "id": "a246aedb", "metadata": {}, "source": [ "The package `torchinfo` provides a `summary()` function that neatly summarizes\n", "this information. We specify the size of the input and see the size\n", - "of each tensor as it passes through layers of the network." + "of each tensor as it passes through layers of the network. " ] }, { "cell_type": "code", "execution_count": 20, - "id": "2f20059e", - "metadata": {}, + "id": "15dd23a9", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [ { "data": { @@ -634,12 +653,12 @@ " input_size=X_train.shape,\n", " col_names=['input_size',\n", " 'output_size',\n", - " 'num_params'])" + " 'num_params'])\n" ] }, { "cell_type": "markdown", - "id": "05a0eb81", + "id": "3cb4b8bc", "metadata": {}, "source": [ "We have truncated the end of the output slightly, here and in subsequent uses.\n", @@ -661,8 +680,10 @@ { "cell_type": "code", "execution_count": 21, - "id": "6ca3030d", - "metadata": {}, + "id": "dae83bc5", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [], "source": [ "X_train_t = torch.tensor(X_train.astype(np.float32))\n", @@ -672,7 +693,7 @@ }, { "cell_type": "markdown", - "id": "7924f53d", + "id": "80475fa2", "metadata": {}, "source": [ "We do the same for the test data." @@ -681,18 +702,18 @@ { "cell_type": "code", "execution_count": 22, - "id": "86723b3e", + "id": "81e217a8", "metadata": {}, "outputs": [], "source": [ "X_test_t = torch.tensor(X_test.astype(np.float32))\n", "Y_test_t = torch.tensor(Y_test.astype(np.float32))\n", - "hit_test = TensorDataset(X_test_t, Y_test_t)" + "hit_test = TensorDataset(X_test_t, Y_test_t)\n" ] }, { "cell_type": "markdown", - "id": "0d055846", + "id": "7f49263d", "metadata": {}, "source": [ "Finally, this dataset is passed to a `DataLoader()` which ultimately\n", @@ -716,7 +737,7 @@ { "cell_type": "code", "execution_count": 23, - "id": "999279fd", + "id": "e5359e31", "metadata": {}, "outputs": [], "source": [ @@ -725,7 +746,7 @@ }, { "cell_type": "markdown", - "id": "9fe60b31", + "id": "105b015f", "metadata": {}, "source": [ "The general training setup in `pytorch_lightning` involves\n", @@ -748,7 +769,7 @@ { "cell_type": "code", "execution_count": 24, - "id": "7bd9cc5c", + "id": "7a19d6d8", "metadata": {}, "outputs": [], "source": [ @@ -756,12 +777,12 @@ " hit_test,\n", " batch_size=32,\n", " num_workers=min(4, max_num_workers),\n", - " validation=hit_test)" + " validation=hit_test)\n" ] }, { "cell_type": "markdown", - "id": "9be1f578", + "id": "8d1f2a76", "metadata": {}, "source": [ "Next we must provide a `pytorch_lightning` module that controls\n", @@ -776,17 +797,17 @@ { "cell_type": "code", "execution_count": 25, - "id": "5be2f822", + "id": "07bc10ef", "metadata": {}, "outputs": [], "source": [ "hit_module = SimpleModule.regression(hit_model,\n", - " metrics={'mae':MeanAbsoluteError()})" + " metrics={'mae':MeanAbsoluteError()})\n" ] }, { "cell_type": "markdown", - "id": "78e707d8", + "id": "1ebf9835", "metadata": {}, "source": [ " By using the `SimpleModule.regression()` method, we indicate that we will use squared-error loss as in\n", @@ -803,7 +824,7 @@ { "cell_type": "code", "execution_count": 26, - "id": "87334d33", + "id": "08c71fb4", "metadata": {}, "outputs": [], "source": [ @@ -812,7 +833,7 @@ }, { "cell_type": "markdown", - "id": "07663639", + "id": "511617c3", "metadata": {}, "source": [ "Finally we are ready to train our model and log the results. We\n", @@ -834,9 +855,9 @@ { "cell_type": "code", "execution_count": 27, - "id": "1a474999", + "id": "81a8c626", "metadata": { - "scrolled": true + "lines_to_next_cell": 0 }, "outputs": [ { @@ -876,7 +897,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "def101fea9464f4b920d238fd076ec39", + "model_id": "ef89ad488fd24d7a8ab978b4d3f22ca1", "version_major": 2, "version_minor": 0 }, @@ -1548,7 +1569,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c193852129d141aa876cc3141f9a92a0", + "model_id": "d44f77bd4233488ebe5c4d84b948f780", "version_major": 2, "version_minor": 0 }, @@ -1562,7 +1583,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "809d7b786e6a4b9f92be6ec878f7bea8", + "model_id": "277ab0a6550f47a587371058c876ce69", "version_major": 2, "version_minor": 0 }, @@ -1576,7 +1597,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ffd4bee67f18480ca919cf5786aa1beb", + "model_id": "778d348b298f403fb26acfd4daf7ca04", "version_major": 2, "version_minor": 0 }, @@ -1606,7 +1627,7 @@ }, { "cell_type": "markdown", - "id": "5e154e47", + "id": "abe1a22c", "metadata": {}, "source": [ "At each step of SGD, the algorithm randomly selects 32 training observations for\n", @@ -1622,13 +1643,15 @@ { "cell_type": "code", "execution_count": 28, - "id": "e3b24643", - "metadata": {}, + "id": "083670c6", + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2a2b2567f1494315ac7eaf368f9c3970", + "model_id": "c22d1cd8cde040988ca2a2718541d22b", "version_major": 2, "version_minor": 0 }, @@ -1640,16 +1663,29 @@ "output_type": "display_data" }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n", - "Runningstage.testing metric DataLoader 0\n", - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n", - " test_loss 125020.2421875\n", - " test_mae 250.8108367919922\n", - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n" - ] + "data": { + "text/html": [ + "
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+       "┃   Runningstage.testing    ┃                           ┃\n",
+       "┃          metric           ┃       DataLoader 0        ┃\n",
+       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+       "│         test_loss         │      125020.2421875       │\n",
+       "│         test_mae          │     250.8108367919922     │\n",
+       "└───────────────────────────┴───────────────────────────┘\n",
+       "
\n" + ], + "text/plain": [ + "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n", + "┃\u001b[1m \u001b[0m\u001b[1m Runningstage.testing \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m┃\n", + "┃\u001b[1m \u001b[0m\u001b[1m metric \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m DataLoader 0 \u001b[0m\u001b[1m \u001b[0m┃\n", + "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n", + "│\u001b[36m \u001b[0m\u001b[36m test_loss \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 125020.2421875 \u001b[0m\u001b[35m \u001b[0m│\n", + "│\u001b[36m \u001b[0m\u001b[36m test_mae \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 250.8108367919922 \u001b[0m\u001b[35m \u001b[0m│\n", + "└───────────────────────────┴───────────────────────────┘\n" + ] + }, + "metadata": {}, + "output_type": "display_data" }, { "data": { @@ -1663,12 +1699,12 @@ } ], "source": [ - "hit_trainer.test(hit_module, datamodule=hit_dm)" + "hit_trainer.test(hit_module, datamodule=hit_dm)\n" ] }, { "cell_type": "markdown", - "id": "0220713b", + "id": "c6b3317b", "metadata": {}, "source": [ "The results of the fit have been logged into a CSV file. We can find the\n", @@ -1684,7 +1720,7 @@ { "cell_type": "code", "execution_count": 29, - "id": "f9b266e7", + "id": "02ba9edf", "metadata": {}, "outputs": [], "source": [ @@ -1693,7 +1729,7 @@ }, { "cell_type": "markdown", - "id": "b7ff26d7", + "id": "daf8ed8d", "metadata": {}, "source": [ "Since we will produce similar plots in later examples, we write a\n", @@ -1703,8 +1739,10 @@ { "cell_type": "code", "execution_count": 30, - "id": "c5752a0b", - "metadata": {}, + "id": "4184557c", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [], "source": [ "def summary_plot(results,\n", @@ -1735,7 +1773,7 @@ }, { "cell_type": "markdown", - "id": "874c2611", + "id": "026c1c4d", "metadata": {}, "source": [ "We now set up our axes, and use our function to produce the MAE plot." @@ -1744,8 +1782,10 @@ { "cell_type": "code", "execution_count": 31, - "id": "ff0c9fa0", - "metadata": {}, + "id": "3244deeb", + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [ { "data": { @@ -1771,7 +1811,7 @@ }, { "cell_type": "markdown", - "id": "212c64a3", + "id": "670a3e8f", "metadata": {}, "source": [ "We can predict directly from the final model, and\n", @@ -1789,8 +1829,10 @@ { "cell_type": "code", "execution_count": 32, - "id": "1ee2b61b", - "metadata": {}, + "id": "36bb545e", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [ { "data": { @@ -1811,21 +1853,32 @@ }, { "cell_type": "markdown", - "id": "64fa7609", + "id": "ae4b9ca6", + "metadata": {}, + "source": [ + " " + ] + }, + { + "cell_type": "markdown", + "id": "b8b7fd3c", "metadata": {}, "source": [ "### Cleanup\n", "In setting up our data module, we had initiated\n", "several worker processes that will remain running.\n", "We delete all references to the torch objects to ensure these processes\n", - "will be killed." + "will be killed.\n", + " " ] }, { "cell_type": "code", "execution_count": 33, - "id": "0ca069a3", - "metadata": {}, + "id": "00371f48", + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [], "source": [ "del(Hitters,\n", @@ -1836,12 +1889,12 @@ " X_test, X_train,\n", " Y_test, Y_train,\n", " X_test_t, Y_test_t,\n", - " hit_trainer, hit_module)" + " hit_trainer, hit_module)\n" ] }, { "cell_type": "markdown", - "id": "ac321b7f", + "id": "5707d61c", "metadata": {}, "source": [ "## Multilayer Network on the MNIST Digit Data\n", @@ -1855,7 +1908,7 @@ { "cell_type": "code", "execution_count": 34, - "id": "af6a0a33", + "id": "3e28d6ba", "metadata": {}, "outputs": [ { @@ -1881,12 +1934,12 @@ " download=True,\n", " transform=ToTensor())\n", " for train in [True, False]]\n", - "mnist_train" + "mnist_train\n" ] }, { "cell_type": "markdown", - "id": "6c8aa040", + "id": "eb8b7e29", "metadata": {}, "source": [ "There are 60,000 images in the training data and 10,000 in the test\n", @@ -1910,7 +1963,7 @@ { "cell_type": "code", "execution_count": 35, - "id": "0df8a0c8", + "id": "cb04829e", "metadata": {}, "outputs": [], "source": [ @@ -1918,12 +1971,12 @@ " mnist_test,\n", " validation=0.2,\n", " num_workers=max_num_workers,\n", - " batch_size=256)" + " batch_size=256)\n" ] }, { "cell_type": "markdown", - "id": "3dc50b18", + "id": "1119e22a", "metadata": {}, "source": [ "Let’s take a look at the data that will get fed into our network. We loop through the first few\n", @@ -1933,8 +1986,10 @@ { "cell_type": "code", "execution_count": 36, - "id": "0c7dd6ee", - "metadata": {}, + "id": "c4a265fc", + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [ { "name": "stdout", @@ -1952,12 +2007,12 @@ " print('X: ', X_.shape)\n", " print('Y: ', Y_.shape)\n", " if idx >= 1:\n", - " break" + " break\n" ] }, { "cell_type": "markdown", - "id": "cc80f21e", + "id": "f65ada90", "metadata": {}, "source": [ "We see that the $X$ for each batch consists of 256 images of size `1x28x28`.\n", @@ -1970,7 +2025,7 @@ { "cell_type": "code", "execution_count": 37, - "id": "63e48c70", + "id": "60339a03", "metadata": {}, "outputs": [], "source": [ @@ -1996,7 +2051,7 @@ }, { "cell_type": "markdown", - "id": "d5a5ffef", + "id": "5b8f87da", "metadata": {}, "source": [ "We see that in the first layer, each `1x28x28` image is flattened, then mapped to\n", @@ -2010,16 +2065,16 @@ { "cell_type": "code", "execution_count": 38, - "id": "92405826", + "id": "1d0f24b2", "metadata": {}, "outputs": [], "source": [ - "mnist_model = MNISTModel()" + "mnist_model = MNISTModel()\n" ] }, { "cell_type": "markdown", - "id": "b16bd23a", + "id": "0ee4771e", "metadata": {}, "source": [ "We can check that the model produces output of expected size based\n", @@ -2029,7 +2084,7 @@ { "cell_type": "code", "execution_count": 39, - "id": "2c8f7a48", + "id": "42a4931b", "metadata": {}, "outputs": [ { @@ -2049,7 +2104,7 @@ }, { "cell_type": "markdown", - "id": "da0de828", + "id": "628d56f9", "metadata": {}, "source": [ "Let’s take a look at the summary of the model. Instead of an `input_size` we can pass\n", @@ -2060,7 +2115,7 @@ { "cell_type": "code", "execution_count": 40, - "id": "9f502df8", + "id": "69e62d88", "metadata": {}, "outputs": [ { @@ -2109,30 +2164,31 @@ }, { "cell_type": "markdown", - "id": "f98b70bb", + "id": "f8e77337", "metadata": {}, "source": [ "Having set up both the model and the data module, fitting this model is\n", "now almost identical to the `Hitters` example. In contrast to our regression model, here we will use the\n", "`SimpleModule.classification()` method which\n", - "uses the cross-entropy loss function instead of mean squared error. It must be supplied with the number of classes in the problem." + "uses the cross-entropy loss function instead of mean squared error. It must be supplied with the number of\n", + "classes in the problem." ] }, { "cell_type": "code", "execution_count": 41, - "id": "d1a7590f", + "id": "2027a378", "metadata": {}, "outputs": [], "source": [ "mnist_module = SimpleModule.classification(mnist_model,\n", " num_classes=10)\n", - "mnist_logger = CSVLogger('logs', name='MNIST')" + "mnist_logger = CSVLogger('logs', name='MNIST')\n" ] }, { "cell_type": "markdown", - "id": "ca665912", + "id": "9146b9b5", "metadata": {}, "source": [ "Now we are ready to go. The final step is to supply training data, and fit the model." @@ -2141,8 +2197,10 @@ { "cell_type": "code", "execution_count": 42, - "id": "e2fe6de3", - "metadata": {}, + "id": "4a5e941d", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [ { "name": "stderr", @@ -2181,7 +2239,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ffdec2122e234eadb8b7396b62cbbe54", + "model_id": "003350959546477780d1c2c8480ac50e", "version_major": 2, "version_minor": 0 }, @@ -2626,12 +2684,12 @@ " logger=mnist_logger,\n", " callbacks=[ErrorTracker()])\n", "mnist_trainer.fit(mnist_module,\n", - " datamodule=mnist_dm)" + " datamodule=mnist_dm)\n" ] }, { "cell_type": "markdown", - "id": "6b4a4e0c", + "id": "8099bdc9", "metadata": {}, "source": [ "We have suppressed the output here, which is a progress report on the\n", @@ -2649,7 +2707,7 @@ }, { "cell_type": "markdown", - "id": "25528d72", + "id": "d6c3bbbc", "metadata": {}, "source": [ "`SimpleModule.classification()` includes\n", @@ -2662,8 +2720,10 @@ { "cell_type": "code", "execution_count": 43, - "id": "73755987", - "metadata": {}, + "id": "603a278a", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [ { "data": { @@ -2685,12 +2745,12 @@ " ylabel='Accuracy')\n", "ax.set_ylim([0.5, 1])\n", "ax.set_ylabel('Accuracy')\n", - "ax.set_xticks(np.linspace(0, 30, 7).astype(int));" + "ax.set_xticks(np.linspace(0, 30, 7).astype(int));\n" ] }, { "cell_type": "markdown", - "id": "ba63a8da", + "id": "654cec05", "metadata": {}, "source": [ "Once again we evaluate the accuracy using the `test()` method of our trainer. This model achieves\n", @@ -2700,13 +2760,13 @@ { "cell_type": "code", "execution_count": 44, - "id": "f5269c40", + "id": "93dc968b", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3e2dcb5833a14d6ebe333941bcf84aa5", + "model_id": "b16d542a27f64cb187d24b941a7f8e96", "version_major": 2, "version_minor": 0 }, @@ -2718,16 +2778,29 @@ "output_type": "display_data" }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n", - "Runningstage.testing metric DataLoader 0\n", - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n", - " test_accuracy 0.9656999707221985\n", - " test_loss 0.15191325545310974\n", - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n" - ] + "data": { + "text/html": [ + "
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+       "┃   Runningstage.testing    ┃                           ┃\n",
+       "┃          metric           ┃       DataLoader 0        ┃\n",
+       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+       "│       test_accuracy       │    0.9656999707221985     │\n",
+       "│         test_loss         │    0.15191325545310974    │\n",
+       "└───────────────────────────┴───────────────────────────┘\n",
+       "
\n" + ], + "text/plain": [ + "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n", + "┃\u001b[1m \u001b[0m\u001b[1m Runningstage.testing \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m┃\n", + "┃\u001b[1m \u001b[0m\u001b[1m metric \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m DataLoader 0 \u001b[0m\u001b[1m \u001b[0m┃\n", + "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n", + "│\u001b[36m \u001b[0m\u001b[36m test_accuracy \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 0.9656999707221985 \u001b[0m\u001b[35m \u001b[0m│\n", + "│\u001b[36m \u001b[0m\u001b[36m test_loss \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 0.15191325545310974 \u001b[0m\u001b[35m \u001b[0m│\n", + "└───────────────────────────┴───────────────────────────┘\n" + ] + }, + "metadata": {}, + "output_type": "display_data" }, { "data": { @@ -2747,7 +2820,7 @@ }, { "cell_type": "markdown", - "id": "bca75042", + "id": "5f7dc344", "metadata": {}, "source": [ "Table 10.1 also reports the error rates resulting from LDA (Chapter 4) and multiclass logistic\n", @@ -2761,7 +2834,7 @@ { "cell_type": "code", "execution_count": 45, - "id": "97a0b304", + "id": "e1975a3e", "metadata": {}, "outputs": [], "source": [ @@ -2782,8 +2855,10 @@ { "cell_type": "code", "execution_count": 46, - "id": "ea685183", - "metadata": {}, + "id": "14c79199", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [ { "name": "stderr", @@ -2824,7 +2899,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "eaf0a438d97e438881c0da4aaa5e08d4", + "model_id": "b5f0bb1fbf824a808eef2d101e91a0ad", "version_major": 2, "version_minor": 0 }, @@ -3272,7 +3347,7 @@ }, { "cell_type": "markdown", - "id": "c376e402", + "id": "f84cfa36", "metadata": {}, "source": [ "We fit the model just as before and compute the test results." @@ -3281,13 +3356,15 @@ { "cell_type": "code", "execution_count": 47, - "id": "c0bd63e3", - "metadata": {}, + "id": "acb6f88d", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "37aac0e4d34244f9b4ecb16eb2deaefa", + "model_id": "30f89591c26d4f9d9d49d589c36baf06", "version_major": 2, "version_minor": 0 }, @@ -3299,16 +3376,29 @@ "output_type": "display_data" }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n", - "Runningstage.testing metric DataLoader 0\n", - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n", - " test_accuracy 0.9235000014305115\n", - " test_loss 0.3271547853946686\n", - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n" - ] + "data": { + "text/html": [ + "
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+       "┃   Runningstage.testing    ┃                           ┃\n",
+       "┃          metric           ┃       DataLoader 0        ┃\n",
+       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+       "│       test_accuracy       │    0.9235000014305115     │\n",
+       "│         test_loss         │    0.3271547853946686     │\n",
+       "└───────────────────────────┴───────────────────────────┘\n",
+       "
\n" + ], + "text/plain": [ + "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n", + "┃\u001b[1m \u001b[0m\u001b[1m Runningstage.testing \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m┃\n", + "┃\u001b[1m \u001b[0m\u001b[1m metric \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m DataLoader 0 \u001b[0m\u001b[1m \u001b[0m┃\n", + "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n", + "│\u001b[36m \u001b[0m\u001b[36m test_accuracy \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 0.9235000014305115 \u001b[0m\u001b[35m \u001b[0m│\n", + "│\u001b[36m \u001b[0m\u001b[36m test_loss \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 0.3271547853946686 \u001b[0m\u001b[35m \u001b[0m│\n", + "└───────────────────────────┴───────────────────────────┘\n" + ] + }, + "metadata": {}, + "output_type": "display_data" }, { "data": { @@ -3328,7 +3418,7 @@ }, { "cell_type": "markdown", - "id": "e0e99a86", + "id": "05244a5e", "metadata": {}, "source": [ "The accuracy is above 90% even for this pretty simple model.\n", @@ -3340,8 +3430,10 @@ { "cell_type": "code", "execution_count": 48, - "id": "6b0d3daa", - "metadata": {}, + "id": "f5b3f811", + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [], "source": [ "del(mnist_test,\n", @@ -3358,7 +3450,7 @@ }, { "cell_type": "markdown", - "id": "1e370989", + "id": "b5b395bb", "metadata": {}, "source": [ "## Convolutional Neural Networks\n", @@ -3369,7 +3461,7 @@ { "cell_type": "code", "execution_count": 49, - "id": "67517b11", + "id": "e1caa7ac", "metadata": {}, "outputs": [ { @@ -3392,7 +3484,7 @@ { "cell_type": "code", "execution_count": 50, - "id": "ee7b040e", + "id": "9e94a7b4", "metadata": {}, "outputs": [], "source": [ @@ -3409,7 +3501,7 @@ }, { "cell_type": "markdown", - "id": "2bba84d8", + "id": "219ccc6f", "metadata": {}, "source": [ "The `CIFAR100` dataset consists of 50,000 training images, each represented by a three-dimensional tensor:\n", @@ -3423,20 +3515,22 @@ { "cell_type": "code", "execution_count": 51, - "id": "bd9e74ad", - "metadata": {}, + "id": "32c78c06", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [], "source": [ "cifar_dm = SimpleDataModule(cifar_train,\n", " cifar_test,\n", " validation=0.2,\n", " num_workers=max_num_workers,\n", - " batch_size=128)" + " batch_size=128)\n" ] }, { "cell_type": "markdown", - "id": "6ee8883b", + "id": "e4570164", "metadata": {}, "source": [ "We again look at the shape of typical batches in our data loaders." @@ -3445,8 +3539,10 @@ { "cell_type": "code", "execution_count": 52, - "id": "a553b926", - "metadata": {}, + "id": "b3c27322", + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [ { "name": "stdout", @@ -3464,16 +3560,16 @@ " print('X: ', X_.shape)\n", " print('Y: ', Y_.shape)\n", " if idx >= 1:\n", - " break" + " break\n" ] }, { "cell_type": "markdown", - "id": "2b25a138", + "id": "f6152280", "metadata": {}, "source": [ "Before we start, we look at some of the training images; similar code produced\n", - "Figure 10.5 on page 445. The example below also illustrates\n", + "Figure 10.5 on page 164. The example below also illustrates\n", "that `TensorDataset` objects can be indexed with integers --- we are choosing\n", "random images from the training data by indexing `cifar_train`. In order to display correctly,\n", "we must reorder the dimensions by a call to `np.transpose()`." @@ -3482,8 +3578,10 @@ { "cell_type": "code", "execution_count": 53, - "id": "94885e68", - "metadata": {}, + "id": "c626e0ff", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [ { "data": { @@ -3508,12 +3606,12 @@ " [1,2,0]),\n", " interpolation=None)\n", " axes[i,j].set_xticks([])\n", - " axes[i,j].set_yticks([])" + " axes[i,j].set_yticks([])\n" ] }, { "cell_type": "markdown", - "id": "04fda6b8", + "id": "2a1c4832", "metadata": {}, "source": [ "Here the `imshow()` method recognizes from the shape of its argument that it is a 3-dimensional array, with the last dimension indexing the three RGB color channels.\n", @@ -3530,7 +3628,7 @@ { "cell_type": "code", "execution_count": 54, - "id": "cdc2528a", + "id": "9d5bcdf3", "metadata": {}, "outputs": [], "source": [ @@ -3549,12 +3647,12 @@ " self.pool = nn.MaxPool2d(kernel_size=(2,2))\n", "\n", " def forward(self, x):\n", - " return self.pool(self.activation(self.conv(x)))" + " return self.pool(self.activation(self.conv(x)))\n" ] }, { "cell_type": "markdown", - "id": "31891282", + "id": "a7204121", "metadata": {}, "source": [ "Notice that we used the `padding = \"same\"` argument to\n", @@ -3575,7 +3673,7 @@ { "cell_type": "code", "execution_count": 55, - "id": "845be1ae", + "id": "3e13e9bc", "metadata": {}, "outputs": [], "source": [ @@ -3597,12 +3695,12 @@ " def forward(self, x):\n", " val = self.conv(x)\n", " val = torch.flatten(val, start_dim=1)\n", - " return self.output(val)" + " return self.output(val)\n" ] }, { "cell_type": "markdown", - "id": "1ed2c036", + "id": "1b07fb1a", "metadata": {}, "source": [ "We build the model and look at the summary. (We had created examples of `X_` earlier.)" @@ -3611,8 +3709,10 @@ { "cell_type": "code", "execution_count": 56, - "id": "be768cbb", - "metadata": {}, + "id": "15c4a382", + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [ { "data": { @@ -3672,7 +3772,7 @@ }, { "cell_type": "markdown", - "id": "6320d294", + "id": "b168c198", "metadata": {}, "source": [ "The total number of trainable parameters is 964,516.\n", @@ -3706,21 +3806,21 @@ { "cell_type": "code", "execution_count": 57, - "id": "67e3e2d9", + "id": "4a40238a", "metadata": {}, "outputs": [], "source": [ "cifar_optimizer = RMSprop(cifar_model.parameters(), lr=0.001)\n", - "cifar_module = SimpleModule.classification(cifar_model, \n", + "cifar_module = SimpleModule.classification(cifar_model,\n", " num_classes=100,\n", " optimizer=cifar_optimizer)\n", - "cifar_logger = CSVLogger('logs', name='CIFAR100')" + "cifar_logger = CSVLogger('logs', name='CIFAR100')\n" ] }, { "cell_type": "code", "execution_count": 58, - "id": "1ea339ff", + "id": "8aab2c62", "metadata": {}, "outputs": [ { @@ -3760,7 +3860,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d9da2a03110a457b95af00983b02eb2a", + "model_id": "08572337f9c44ae08b65fca1a08db4ce", "version_major": 2, "version_minor": 0 }, @@ -4205,12 +4305,12 @@ " logger=cifar_logger,\n", " callbacks=[ErrorTracker()])\n", "cifar_trainer.fit(cifar_module,\n", - " datamodule=cifar_dm)" + " datamodule=cifar_dm)\n" ] }, { "cell_type": "markdown", - "id": "977e27d9", + "id": "090f6400", "metadata": {}, "source": [ "This model can take 10 minutes or more to run and achieves about 42% accuracy on the test\n", @@ -4226,8 +4326,10 @@ { "cell_type": "code", "execution_count": 59, - "id": "0a9068d9", - "metadata": {}, + "id": "12474ef6", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [ { "data": { @@ -4255,7 +4357,7 @@ }, { "cell_type": "markdown", - "id": "9c48006a", + "id": "f3fe0cd4", "metadata": {}, "source": [ "Finally, we evaluate our model on our test data." @@ -4264,13 +4366,15 @@ { "cell_type": "code", "execution_count": 60, - "id": "e71eff9b", - "metadata": {}, + "id": "9d632437", + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "62b123ff8d5a4d4daf9b560610320546", + "model_id": "f8918dba5b82493399bab84a5d89568d", "version_major": 2, "version_minor": 0 }, @@ -4282,16 +4386,29 @@ "output_type": "display_data" }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n", - "Runningstage.testing metric DataLoader 0\n", - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n", - " test_accuracy 0.42969998717308044\n", - " test_loss 2.4235761165618896\n", - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n" - ] + "data": { + "text/html": [ + "
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+       "┃   Runningstage.testing    ┃                           ┃\n",
+       "┃          metric           ┃       DataLoader 0        ┃\n",
+       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+       "│       test_accuracy       │    0.42969998717308044    │\n",
+       "│         test_loss         │    2.4235761165618896     │\n",
+       "└───────────────────────────┴───────────────────────────┘\n",
+       "
\n" + ], + "text/plain": [ + "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n", + "┃\u001b[1m \u001b[0m\u001b[1m Runningstage.testing \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m┃\n", + "┃\u001b[1m \u001b[0m\u001b[1m metric \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m DataLoader 0 \u001b[0m\u001b[1m \u001b[0m┃\n", + "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n", + "│\u001b[36m \u001b[0m\u001b[36m test_accuracy \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 0.42969998717308044 \u001b[0m\u001b[35m \u001b[0m│\n", + "│\u001b[36m \u001b[0m\u001b[36m test_loss \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 2.4235761165618896 \u001b[0m\u001b[35m \u001b[0m│\n", + "└───────────────────────────┴───────────────────────────┘\n" + ] + }, + "metadata": {}, + "output_type": "display_data" }, { "data": { @@ -4306,12 +4423,12 @@ ], "source": [ "cifar_trainer.test(cifar_module,\n", - " datamodule=cifar_dm)" + " datamodule=cifar_dm)\n" ] }, { "cell_type": "markdown", - "id": "2d03b891", + "id": "4b69e259", "metadata": {}, "source": [ "### Hardware Acceleration\n", @@ -4332,8 +4449,10 @@ { "cell_type": "code", "execution_count": 61, - "id": "58eae430", - "metadata": {}, + "id": "52a43158", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [ { "name": "stderr", @@ -4372,7 +4491,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "df1f8b9e9a05488288d5f8b417bd2378", + "model_id": "6e1d5e16e74643259376abbb16295d08", "version_major": 2, "version_minor": 0 }, @@ -4821,7 +4940,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7d75c270a4a84b35bb2ed7870951db56", + "model_id": "3cf62ed2c144476bb6b851b230ebe335", "version_major": 2, "version_minor": 0 }, @@ -4850,7 +4969,7 @@ }, { "cell_type": "markdown", - "id": "0c7227ef", + "id": "b72de871", "metadata": {}, "source": [ "This yields approximately two- or three-fold acceleration for each epoch.\n", @@ -4860,7 +4979,7 @@ }, { "cell_type": "markdown", - "id": "4328a190", + "id": "b86d6ef4", "metadata": {}, "source": [ "## Using Pretrained CNN Models\n", @@ -4882,8 +5001,10 @@ { "cell_type": "code", "execution_count": 62, - "id": "638a1c8c", - "metadata": {}, + "id": "db2febe9", + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [ { "data": { @@ -4910,7 +5031,7 @@ }, { "cell_type": "markdown", - "id": "a2c5aec2", + "id": "f6696ee2", "metadata": {}, "source": [ "We now set up the trained network with the weights we read in code block~6. The model has 50 layers, with a fair bit of complexity." @@ -4919,8 +5040,10 @@ { "cell_type": "code", "execution_count": 63, - "id": "7776ed1e", - "metadata": {}, + "id": "2aabd724", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [ { "data": { @@ -5127,12 +5250,12 @@ " input_data=imgs,\n", " col_names=['input_size',\n", " 'output_size',\n", - " 'num_params'])" + " 'num_params'])\n" ] }, { "cell_type": "markdown", - "id": "0d29ab45", + "id": "c4ee6ebe", "metadata": {}, "source": [ "We set the mode to `eval()` to ensure that the model is ready to predict on new data." @@ -5141,8 +5264,10 @@ { "cell_type": "code", "execution_count": 64, - "id": "5c8e359d", - "metadata": {}, + "id": "6d27342d", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [ { "data": { @@ -5336,7 +5461,7 @@ }, { "cell_type": "markdown", - "id": "228a6411", + "id": "3c1fbc71", "metadata": {}, "source": [ "Inspecting the output above, we see that when setting up the\n", @@ -5349,16 +5474,16 @@ { "cell_type": "code", "execution_count": 65, - "id": "80f19869", + "id": "efdbeda1", "metadata": {}, "outputs": [], "source": [ - "img_preds = resnet_model(imgs)" + "img_preds = resnet_model(imgs)\n" ] }, { "cell_type": "markdown", - "id": "e7bd653f", + "id": "38620865", "metadata": {}, "source": [ "Let’s look at the predicted probabilities for each of the top 3 choices. First we compute\n", @@ -5370,17 +5495,17 @@ { "cell_type": "code", "execution_count": 66, - "id": "6892597d", + "id": "82ceab1c", "metadata": {}, "outputs": [], "source": [ "img_probs = np.exp(np.asarray(img_preds.detach()))\n", - "img_probs /= img_probs.sum(1)[:,None]" + "img_probs /= img_probs.sum(1)[:,None]\n" ] }, { "cell_type": "markdown", - "id": "b0f07763", + "id": "0e3ae755", "metadata": {}, "source": [ "In order to see the class labels, we must download the index file associated with `imagenet`. {This is avalable from the book website and [s3.amazonaws.com/deep-learning-models/image-models/imagenet_class_index.json](https://s3.amazonaws.com/deep-learning-models/image-models/imagenet_class_index.json).}" @@ -5389,7 +5514,7 @@ { "cell_type": "code", "execution_count": 67, - "id": "3eb1e1b0", + "id": "921ee168", "metadata": {}, "outputs": [], "source": [ @@ -5398,12 +5523,12 @@ " labs.items()],\n", " columns=['idx', 'label'])\n", "class_labels = class_labels.set_index('idx')\n", - "class_labels = class_labels.sort_index()" + "class_labels = class_labels.sort_index()\n" ] }, { "cell_type": "markdown", - "id": "7779e914", + "id": "9fdd7587", "metadata": {}, "source": [ "We’ll now construct a data frame for each image file\n", @@ -5414,8 +5539,10 @@ { "cell_type": "code", "execution_count": 68, - "id": "0e89d251", - "metadata": {}, + "id": "e7770017", + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [ { "name": "stdout", @@ -5460,12 +5587,12 @@ " img_df['prob'] = img_probs[i]\n", " img_df = img_df.sort_values(by='prob', ascending=False)[:3]\n", " print(f'Image: {imgfile}')\n", - " print(img_df.reset_index().drop(columns=['idx']))" + " print(img_df.reset_index().drop(columns=['idx']))\n" ] }, { "cell_type": "markdown", - "id": "7f9d3843", + "id": "12862208", "metadata": {}, "source": [ "We see that the model\n", @@ -5478,8 +5605,10 @@ { "cell_type": "code", "execution_count": 69, - "id": "69456669", - "metadata": {}, + "id": "af2f9856", + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [], "source": [ "del(cifar_test,\n", @@ -5493,7 +5622,7 @@ }, { "cell_type": "markdown", - "id": "64b33c4d", + "id": "d9e8d59c", "metadata": {}, "source": [ "## IMDB Document Classification\n", @@ -5513,14 +5642,17 @@ "* `load_sparse()`, a sparse matrix version usable by `sklearn`, since we will compare with a lasso fit;\n", "* `load_sequential()`, a padded\n", "version of the original sequence representation, limited to the last\n", - "500 words of each review." + "500 words of each review.\n", + "\n" ] }, { "cell_type": "code", "execution_count": 70, - "id": "70b3ece2", - "metadata": {}, + "id": "afd98123", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [ { "data": { @@ -5539,12 +5671,12 @@ " imdb_seq_test) = load_sequential(root='data/IMDB')\n", "padded_sample = np.asarray(imdb_seq_train.tensors[0][0])\n", "sample_review = padded_sample[padded_sample > 0][:12]\n", - "sample_review[:12]" + "sample_review[:12]\n" ] }, { "cell_type": "markdown", - "id": "d6b73c18", + "id": "28a01855", "metadata": {}, "source": [ "The datasets `imdb_seq_train` and `imdb_seq_test` are\n", @@ -5562,7 +5694,7 @@ { "cell_type": "code", "execution_count": 71, - "id": "349b2af7", + "id": "5981eb05", "metadata": {}, "outputs": [ { @@ -5583,7 +5715,7 @@ }, { "cell_type": "markdown", - "id": "99d5a108", + "id": "b579cb29", "metadata": {}, "source": [ "For our first model, we have created a binary feature for each\n", @@ -5596,8 +5728,10 @@ { "cell_type": "code", "execution_count": 72, - "id": "ffa8da37", - "metadata": {}, + "id": "f08024ba", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [], "source": [ "max_num_workers=10\n", @@ -5607,12 +5741,12 @@ " imdb_test,\n", " validation=2000,\n", " num_workers=min(6, max_num_workers),\n", - " batch_size=512)" + " batch_size=512)\n" ] }, { "cell_type": "markdown", - "id": "e53eea99", + "id": "c070a319", "metadata": {}, "source": [ "We’ll use a two-layer model for our first model." @@ -5621,8 +5755,10 @@ { "cell_type": "code", "execution_count": 73, - "id": "11eeeabb", - "metadata": {}, + "id": "7fd34906", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [], "source": [ "class IMDBModel(nn.Module):\n", @@ -5642,12 +5778,12 @@ " self.activation,\n", " self.output]:\n", " val = _map(val)\n", - " return torch.flatten(val)" + " return torch.flatten(val)\n" ] }, { "cell_type": "markdown", - "id": "943bf8d4", + "id": "fcc2a8af", "metadata": {}, "source": [ "We now instantiate our model and look at a summary." @@ -5656,7 +5792,7 @@ { "cell_type": "code", "execution_count": 74, - "id": "5fe55a29", + "id": "56f74fdb", "metadata": {}, "outputs": [ { @@ -5695,12 +5831,12 @@ " input_size=imdb_test.tensors[0].size(),\n", " col_names=['input_size',\n", " 'output_size',\n", - " 'num_params'])" + " 'num_params'])\n" ] }, { "cell_type": "markdown", - "id": "a49a0145", + "id": "a32aca43", "metadata": {}, "source": [ "We’ll again use\n", @@ -5718,19 +5854,19 @@ { "cell_type": "code", "execution_count": 75, - "id": "e1ebbc70", + "id": "3da7e0bc", "metadata": {}, "outputs": [], "source": [ "imdb_optimizer = RMSprop(imdb_model.parameters(), lr=0.001)\n", "imdb_module = SimpleModule.binary_classification(\n", " imdb_model,\n", - " optimizer=imdb_optimizer)" + " optimizer=imdb_optimizer)\n" ] }, { "cell_type": "markdown", - "id": "18076d4c", + "id": "940c8342", "metadata": {}, "source": [ "Having loaded the datasets into a data module\n", @@ -5741,7 +5877,7 @@ { "cell_type": "code", "execution_count": 76, - "id": "9feddeb2", + "id": "3b6de185", "metadata": {}, "outputs": [ { @@ -5789,7 +5925,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "cc447577a88b4320b9f6a755a35e585b", + "model_id": "1b539194264149abbe1b411d90c66946", "version_major": 2, "version_minor": 0 }, @@ -6240,7 +6376,7 @@ }, { "cell_type": "markdown", - "id": "f28e6658", + "id": "5985c44a", "metadata": {}, "source": [ "Evaluating the test error yields roughly 86% accuracy." @@ -6249,13 +6385,15 @@ { "cell_type": "code", "execution_count": 77, - "id": "887d7dad", - "metadata": {}, + "id": "97f86a32", + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "29c608e896a54d4ca80dcfc77312ebcc", + "model_id": "2e3673c6298342d79c32888a28e2c9cd", "version_major": 2, "version_minor": 0 }, @@ -6267,16 +6405,29 @@ "output_type": "display_data" }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n", - "Runningstage.testing metric DataLoader 0\n", - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n", - " test_accuracy 0.8492799997329712\n", - " test_loss 1.2849037647247314\n", - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n" - ] + "data": { + "text/html": [ + "
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+       "┃   Runningstage.testing    ┃                           ┃\n",
+       "┃          metric           ┃       DataLoader 0        ┃\n",
+       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+       "│       test_accuracy       │    0.8492799997329712     │\n",
+       "│         test_loss         │    1.2849037647247314     │\n",
+       "└───────────────────────────┴───────────────────────────┘\n",
+       "
\n" + ], + "text/plain": [ + "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n", + "┃\u001b[1m \u001b[0m\u001b[1m Runningstage.testing \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m┃\n", + "┃\u001b[1m \u001b[0m\u001b[1m metric \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m DataLoader 0 \u001b[0m\u001b[1m \u001b[0m┃\n", + "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n", + "│\u001b[36m \u001b[0m\u001b[36m test_accuracy \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 0.8492799997329712 \u001b[0m\u001b[35m \u001b[0m│\n", + "│\u001b[36m \u001b[0m\u001b[36m test_loss \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 1.2849037647247314 \u001b[0m\u001b[35m \u001b[0m│\n", + "└───────────────────────────┴───────────────────────────┘\n" + ] + }, + "metadata": {}, + "output_type": "display_data" }, { "data": { @@ -6296,7 +6447,7 @@ }, { "cell_type": "markdown", - "id": "d2f87545", + "id": "3d677b24", "metadata": {}, "source": [ "### Comparison to Lasso\n", @@ -6309,7 +6460,7 @@ { "cell_type": "code", "execution_count": 78, - "id": "055a9aeb", + "id": "e36e1542", "metadata": {}, "outputs": [], "source": [ @@ -6317,12 +6468,12 @@ " (X_valid, Y_valid),\n", " (X_test, Y_test)) = load_sparse(validation=2000,\n", " random_state=0,\n", - " root='data/IMDB')" + " root='data/IMDB')\n" ] }, { "cell_type": "markdown", - "id": "d8324d03", + "id": "8216f2e7", "metadata": {}, "source": [ "Similar to what we did in\n", @@ -6333,43 +6484,47 @@ { "cell_type": "code", "execution_count": 79, - "id": "e3e1e181", - "metadata": {}, + "id": "ee6d6859", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [], "source": [ "lam_max = np.abs(X_train.T * (Y_train - Y_train.mean())).max()\n", "lam_val = lam_max * np.exp(np.linspace(np.log(1),\n", - " np.log(1e-4), 50))" + " np.log(1e-4), 50))\n" ] }, { "cell_type": "markdown", - "id": "73e7d53e", + "id": "cf9b703f", "metadata": {}, "source": [ "With `LogisticRegression()` the regularization parameter\n", "$C$ is specified as the inverse of $\\lambda$. There are several\n", "solvers for logistic regression; here we use `liblinear` which\n", - "works well with the sparse input format." + "works well with the sparse input format. " ] }, { "cell_type": "code", "execution_count": 80, - "id": "dd64350c", - "metadata": {}, + "id": "35a6e3c2", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [], "source": [ "logit = LogisticRegression(penalty='l1', \n", " C=1/lam_max,\n", " solver='liblinear',\n", " warm_start=True,\n", - " fit_intercept=True)" + " fit_intercept=True)\n" ] }, { "cell_type": "markdown", - "id": "f0e9739a", + "id": "49cec308", "metadata": {}, "source": [ "The path of 50 values takes approximately 40 seconds to run." @@ -6378,7 +6533,7 @@ { "cell_type": "code", "execution_count": 81, - "id": "0bf8b868", + "id": "a8943a6a", "metadata": {}, "outputs": [], "source": [ @@ -6389,12 +6544,12 @@ " logit.C = 1/l\n", " logit.fit(X_train, Y_train)\n", " coefs.append(logit.coef_.copy())\n", - " intercepts.append(logit.intercept_)" + " intercepts.append(logit.intercept_)\n" ] }, { "cell_type": "markdown", - "id": "95f4fbff", + "id": "4f20738d", "metadata": {}, "source": [ "The coefficient and intercepts have an extraneous dimension which can be removed\n", @@ -6404,17 +6559,19 @@ { "cell_type": "code", "execution_count": 82, - "id": "28256828", - "metadata": {}, + "id": "08268ae5", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [], "source": [ "coefs = np.squeeze(coefs)\n", - "intercepts = np.squeeze(intercepts)" + "intercepts = np.squeeze(intercepts)\n" ] }, { "cell_type": "markdown", - "id": "94e746f6", + "id": "94991ca8", "metadata": {}, "source": [ "We’ll now make a plot to compare our neural network results with the\n", @@ -6424,8 +6581,10 @@ { "cell_type": "code", "execution_count": 83, - "id": "a5945618", - "metadata": {}, + "id": "e98f4506", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [], "source": [ "%%capture\n", @@ -6449,12 +6608,12 @@ " label=data_)\n", "axes[0].legend()\n", "axes[0].set_xlabel(r'$-\\log(\\lambda)$', fontsize=20)\n", - "axes[0].set_ylabel('Accuracy', fontsize=20)" + "axes[0].set_ylabel('Accuracy', fontsize=20)\n" ] }, { "cell_type": "markdown", - "id": "7dc7e494", + "id": "aac5c38c", "metadata": {}, "source": [ "Notice the use of `%%capture`, which suppresses the displaying of the partially completed figure. This is useful\n", @@ -6465,8 +6624,10 @@ { "cell_type": "code", "execution_count": 84, - "id": "74704884", - "metadata": {}, + "id": "6925d53a", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [ { "data": { @@ -6499,7 +6660,7 @@ }, { "cell_type": "markdown", - "id": "7898334d", + "id": "2a8b7736", "metadata": {}, "source": [ "From the graphs we see that the accuracy of the lasso logistic regression peaks at about $0.88$, as it does for the neural network.\n", @@ -6510,8 +6671,10 @@ { "cell_type": "code", "execution_count": 85, - "id": "b1ecfca8", - "metadata": {}, + "id": "6d1f7885", + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [], "source": [ "del(imdb_model,\n", @@ -6524,7 +6687,7 @@ }, { "cell_type": "markdown", - "id": "0169cd1b", + "id": "4f79c23a", "metadata": {}, "source": [ "## Recurrent Neural Networks\n", @@ -6534,7 +6697,7 @@ }, { "cell_type": "markdown", - "id": "6e7f96cc", + "id": "3d376496", "metadata": {}, "source": [ "### Sequential Models for Document Classification\n", @@ -6548,13 +6711,13 @@ "`ISLP` library. Notably, since more than 90% of the documents\n", "had fewer than 500 words, we set the document length to 500. For\n", "longer documents, we used the last 500 words, and for shorter\n", - "documents, we padded the front with blanks." + "documents, we padded the front with blanks.\n" ] }, { "cell_type": "code", "execution_count": 86, - "id": "5503cb3f", + "id": "b5c2b72c", "metadata": {}, "outputs": [], "source": [ @@ -6563,12 +6726,12 @@ " validation=2000,\n", " batch_size=300,\n", " num_workers=min(6, max_num_workers)\n", - " )" + " )\n" ] }, { "cell_type": "markdown", - "id": "356b76f2", + "id": "4d1b3ef1", "metadata": {}, "source": [ "The first layer of the RNN is an embedding layer of size 32, which will be\n", @@ -6585,7 +6748,7 @@ }, { "cell_type": "markdown", - "id": "14ddd9a9", + "id": "fe55c7fa", "metadata": {}, "source": [ "The second layer is an LSTM with 32 units, and the output\n", @@ -6597,8 +6760,10 @@ { "cell_type": "code", "execution_count": 87, - "id": "8b683641", - "metadata": {}, + "id": "8985cdb1", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [], "source": [ "class LSTMModel(nn.Module):\n", @@ -6616,7 +6781,7 @@ }, { "cell_type": "markdown", - "id": "b7903383", + "id": "78fc988d", "metadata": {}, "source": [ "We instantiate and take a look at the summary of the model, using the\n", @@ -6626,7 +6791,7 @@ { "cell_type": "code", "execution_count": 88, - "id": "702aa8de", + "id": "79184187", "metadata": {}, "outputs": [ { @@ -6663,12 +6828,12 @@ " input_data=imdb_seq_train.tensors[0][:10],\n", " col_names=['input_size',\n", " 'output_size',\n", - " 'num_params'])" + " 'num_params'])\n" ] }, { "cell_type": "markdown", - "id": "f663b95c", + "id": "d5b2c85e", "metadata": {}, "source": [ "The 10,003 is suppressed in the summary, but we see it in the\n", @@ -6678,19 +6843,21 @@ { "cell_type": "code", "execution_count": 89, - "id": "e48aa272", + "id": "29036a47", "metadata": {}, "outputs": [], "source": [ "lstm_module = SimpleModule.binary_classification(lstm_model)\n", - "lstm_logger = CSVLogger('logs', name='IMDB_LSTM')" + "lstm_logger = CSVLogger('logs', name='IMDB_LSTM')\n" ] }, { "cell_type": "code", "execution_count": 90, - "id": "143be7e8", - "metadata": {}, + "id": "0a31576a", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [ { "name": "stderr", @@ -6729,7 +6896,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "1a4cf448a84d4046a07c885d92a3e0c0", + "model_id": "903e15e146454c9c83ea87b85af4a97b", "version_major": 2, "version_minor": 0 }, @@ -7034,12 +7201,12 @@ " logger=lstm_logger,\n", " callbacks=[ErrorTracker()])\n", "lstm_trainer.fit(lstm_module,\n", - " datamodule=imdb_seq_dm)" + " datamodule=imdb_seq_dm)\n" ] }, { "cell_type": "markdown", - "id": "55dc2b66", + "id": "a20a2113", "metadata": {}, "source": [ "The rest is now similar to other networks we have fit. We\n", @@ -7049,13 +7216,13 @@ { "cell_type": "code", "execution_count": 91, - "id": "9272e2ba", + "id": "b2602a79", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "feae6249db0d4706a68bf195a315d97a", + "model_id": "6f9781f37c4d428da10cd12b67dac6fb", "version_major": 2, "version_minor": 0 }, @@ -7067,21 +7234,34 @@ "output_type": "display_data" }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n", - "Runningstage.testing metric DataLoader 0\n", - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n", - " test_accuracy 0.8339200019836426\n", - " test_loss 0.6944115161895752\n", - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n" - ] + "data": { + "text/html": [ + "
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+       "┃   Runningstage.testing    ┃                           ┃\n",
+       "┃          metric           ┃       DataLoader 0        ┃\n",
+       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+       "│       test_accuracy       │    0.8501600027084351     │\n",
+       "│         test_loss         │    0.8030509352684021     │\n",
+       "└───────────────────────────┴───────────────────────────┘\n",
+       "
\n" + ], + "text/plain": [ + "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n", + "┃\u001b[1m \u001b[0m\u001b[1m Runningstage.testing \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m┃\n", + "┃\u001b[1m \u001b[0m\u001b[1m metric \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m DataLoader 0 \u001b[0m\u001b[1m \u001b[0m┃\n", + "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n", + "│\u001b[36m \u001b[0m\u001b[36m test_accuracy \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 0.8501600027084351 \u001b[0m\u001b[35m \u001b[0m│\n", + "│\u001b[36m \u001b[0m\u001b[36m test_loss \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 0.8030509352684021 \u001b[0m\u001b[35m \u001b[0m│\n", + "└───────────────────────────┴───────────────────────────┘\n" + ] + }, + "metadata": {}, + "output_type": "display_data" }, { "data": { "text/plain": [ - "[{'test_loss': 0.6944115161895752, 'test_accuracy': 0.8339200019836426}]" + "[{'test_loss': 0.8030509352684021, 'test_accuracy': 0.8501600027084351}]" ] }, "execution_count": 91, @@ -7095,7 +7275,7 @@ }, { "cell_type": "markdown", - "id": "eee55195", + "id": "54857d49", "metadata": {}, "source": [ "We once again show the learning progress, followed by cleanup." @@ -7104,8 +7284,10 @@ { "cell_type": "code", "execution_count": 92, - "id": "5d9d387e", - "metadata": {}, + "id": "32c3e3da", + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [ { "data": { @@ -7119,7 +7301,7 @@ }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiEAAAISCAYAAAAa+R+EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABBA0lEQVR4nO3dfXhT9f3/8Vca2tKCLfdtaSoFRUTlRrmp6OpA0aobA0s3BJXKVKZfxCLzUlEQcZtcU6dFZTr9IuwOZNTA/P5UHNaiVVEcDNSJKIpQsC2i0tqqLabn90fXjNAWmvYkn5z2+biuXJCTk+QdQnJe+ZzPjcuyLEsAAABhFmW6AAAA0DERQgAAgBGEEAAAYAQhBAAAGEEIAQAARhBCAACAEYQQAABgBCEEAAAYQQgBAABGEEIAAIARRkPIq6++qgkTJqhv375yuVxat27dce+zceNGnXXWWYqNjdXJJ5+sFStWhLxOAABgP6MhpLq6WsOGDdPSpUtbtP/u3bv1ox/9SOPGjdO2bds0Z84cXXvttXrxxRdDXCkAALCbK1IWsHO5XFq7dq0mTZrU7D633XabnnvuOb333nv+bZdffrkOHTqk9evXh6FKAABgl06mCwjGpk2bNH78+IBtWVlZmjNnTrP3qampUU1Njf96XV2dvvzyS/Xs2VMulytUpQIA0O5YlqWvv/5affv2VVRU20+mOCqElJWVKSkpKWBbUlKSKisr9e233youLq7RfRYvXqxFixaFq0QAANq9kpISeTyeNj+Oo0JIa8ybN09z5871X6+oqNCJJ56okpISJSQkGKwMAABnqaysVFpamk444QRbHs9RISQ5OVnl5eUB28rLy5WQkNBkK4gkxcbGKjY2ttH2hIQEQggAAK1gV3cGR80TMmbMGBUWFgZs27Bhg8aMGWOoIgAA0FpGQ0hVVZW2bdumbdu2Saofgrtt2zbt3btXUv2plOnTp/v3v/766/XJJ5/o1ltv1QcffKDf//73+tvf/qabb77ZRPkAAKANjJ6O+ec//6lx48b5rzf03cjNzdWKFStUWlrqDySS1L9/fz333HO6+eabtWTJEnk8Hv3v//6vsrKywl47ACB4Pp9PxcXFKi0tVUpKijIzM+V2u02XBUMiZp6QcKmsrFRiYqIqKiqa7RNiWZa+//57+Xy+MFeHUHC73erUqRNDsgHDvF6v8vLytG/fPv82j8ejJUuWKDs722Blx0Zw+q+WHEOD4aiOqeFQW1ur0tJSffPNN6ZLgY3i4+OVkpKimJgY06UAHZLX61VOTo6O/t27f/9+5eTkqKCgICKDiFODk1PQEnKEuro6ffTRR3K73erdu7diYmL49exwlmWptrZWn3/+uXw+nwYOHGjLBDsAWs7n8yk9PT3gQH4kl8slj8ej3bt3R1QLQ3PBqeG4EKnBKZRoCQmh2tpa1dXVKS0tTfHx8abLgU3i4uIUHR2tPXv2qLa2Vp07dzZdEtChFBcXNxtApPofCyUlJSouLtbYsWNb9Rx2nzLx+XzKy8trFEAa6nW5XJozZ44mTpwYUcHJafhJ2AR+Kbc/vKdAcHw+nzZu3KhVq1Zp48aNbeojV1paaut+R/N6vUpPT9e4ceM0bdo0jRs3Tunp6fJ6va16PCm44ITW45sZABDA7oN6SkqKrfsdqeGUydGBoaGvSWtrDnVwQj1CCADALxQH9czMTHk8nmb72LlcLqWlpSkzMzOoxz3eKRNJmjNnTqtacUIZnPBfhJAQsbMp05T09HTl5+e3eP+NGzfK5XLp0KFDIasJQCA7v2tCdVB3u91asmSJpMbTfTdcz8/PD7pvRShPmYQqOCEQISQEQnF+8lhcLtcxL3fffXerHvftt9/WzJkzW7z/Oeeco9LSUiUmJrbq+YD2LBQ/TOz+rgnlQT07O1sFBQVKTU0N2O7xeFo9yiSUp0xCFZxwFKuDqaiosCRZFRUVjW779ttvrffff9/69ttvW/34zzzzjOVyuSxJAReXy2W5XC7rmWeeaUv5TSotLfVf8vPzrYSEhIBtX3/9tX/furo66/Dhw7bXEOnseG+B1nrmmWcsj8cT8J3g8Xja9H0Qiu+alStXNnq8pi4rV65sdd3ff/+9VVRUZK1cudIqKiqyvv/++1Y/VlFRUYvqLSoqavVzNPXepaWl2fJdbue/Rbgc6xjaGoSQIzR1oKqrq7OqqqpadKmoqLBSU1Ob/SC4XC7L4/FYFRUVLXq8urq6oF/f8uXLrcTERP/1hg/p888/b5111llWdHS0VVRUZO3atcv6yU9+YvXp08fq0qWLNXLkSGvDhg0Bj9WvXz/roYce8l+XZD355JPWpEmTrLi4OOvkk0+2/v73vzd6rq+++iqglvXr11unnnqq1aVLFysrK8v67LPP/Pc5fPiwNXv2bCsxMdHq0aOHdeutt1rTp0+3Jk6cGPRrPxZCCFrK7gNDKMLC999/3+jAePRjp6WlBV17OA7qdmr4d2jq37ct/w5NPY/dYSEUwTQcCCFtFGwIqaqqatGHMhSXqqqqoF9fcyFk6NCh1j/+8Q9r165d1hdffGFt27bNevzxx613333X+vDDD6358+dbnTt3tvbs2eO/b1MhxOPxWCtXrrQ++ugj66abbrK6du1qffHFFwHPdWQIiY6OtsaPH2+9/fbb1pYtW6zBgwdb06ZN8z/mr3/9a6tHjx6W1+u1duzYYV1//fVWQkICIQRG2H1gcFpYCNdB3U4NIe/omkPZ+txWJlrM7UIIaaOOGkLWrVt33Puefvrp1iOPPOK/3lQImT9/fqN/mxdeeCHguY4MIZKsXbt2+e+zdOlSKykpyX89KSnJuv/++/3Xv//+e+vEE08khCDsQnFgCFVYCOVpE6ce1EN1ysRuoQqm4WJ3CKFj6nHEx8erqqqqRZfnn3++RY/5/PPPt+jx7Jy1deTIkQHXq6qqdMstt2jw4MHq1q2bunbtqh07dgSsWtyUoUOH+v/epUsXJSQk6MCBA83uHx8fr5NOOsl/PSUlxb9/RUWFysvLNXr0aP/tbrdbI0aMCOq1AW0VqlEhoeo4Gcrho6HoQBpq2dnZ+vTTT1VUVKSVK1eqqKhIu3fvjshawzEJmpNGZzJt+3G4XC516dKlRftedNFF8ng82r9/f5NfZg3rI1x00UVh71F99Gu45ZZbtGHDBj3wwAM6+eSTFRcXp5ycHNXW1h7zcaKjowOuu1wu1dXVBbV/U/82gEmhmlY8VGGhYfjo8b5rWjt8NDs7WxMnTnTUyrFut7vVU76HUzhmj3XSgnu0hNjISUO6Xn/9dV199dW67LLLNGTIECUnJ+vTTz8Naw2JiYlKSkrS22+/7d/m8/m0devWsNYBhOrAEKq5JsLxXdNwUJ86darGjh0bEd9b7YETZ48NJUKIzZzSlDlw4EB5vV5t27ZN27dv17Rp047ZohEqs2fP1uLFi/X3v/9dO3fuVF5enr766itWL8Yx2d3cHKoDQyjDglO+axDIibPHhhIhJASccH7ywQcfVPfu3XXOOedowoQJysrK0llnnRX2Om677TZNnTpV06dP15gxY9S1a1dlZWWx0i2aFYrJAEM5O2Yow4ITvmsQyImzx4aULd1bHSTUk5WhbXw+n3XKKacEjMKxA+9t+xDKoY2hHhXixImpEDp2j+gJx0RzlmX/6Bg6psKoPXv26B//+Id++MMfqqamRo8++qh2796tadOmmS4NEeZ4zc0ul0tz5szRxIkT23R6o6lOffn5+W1uXXBKx0mEh92df5264B4hBEZFRUVpxYoVuuWWW2RZls444wy99NJLGjx4sOnSEGFCNYLlSE4cFQLnsjOYhnrEVKgQQmBUWlqaXn/9ddNlwAFCPbSxAS0WcKKGviY5OTmNpkKItNGZR6JjKgBHcGpzMxAuThwxRUsIAEdwanMzEE5OO6VICAEQMj6fz7YvQ6c2NwPh5qRTipyOARASoZjPw4nNzQCaR0sIANs1TB999GmThumj2xIYnNbcDKB5hBAAtgr1fB6Ss5qbATSP0zGh4vNJGzdKq1bV/xlh8/UfbezYsZozZ47/enp6uvLz8495H5fLpXXr1rX5ue16HLSenWuxOHb6aABhRwgJBa9XSk+Xxo2Tpk2r/zM9vX57CEyYMEEXX3xxk7cVFxfL5XLpnXfeCeox3377bc2cOdOO8vzuvvtuDR8+vNH20tJSXXLJJbY+F1rO7r4b4ZrPA4DzEULs5vVKOTnS0b8E9++v3x6CIHLNNddow4YNTf76XL58uUaOHKmhQ4cG9Zi9e/dWfHy8XSUeU3JysmJjY8PyXAgUiqW/mc8DQEsRQo7HsqTq6pZdKiulm26qv09TjyNJeXn1+7Xk8Zp6nCb8+Mc/Vu/evbVixYqA7VVVVVqzZo0mTZqkqVOnKjU1VfHx8RoyZIhWrVp1zMc8+nTMRx99pPPOO0+dO3fWaaedpg0bNjS6z2233aZTTjlF8fHxGjBggBYsWKDDhw9LklasWKFFixZp+/btcrlccrlc/nqPPh3z7rvv6vzzz1dcXJx69uypmTNnqqqqyn/71VdfrUmTJumBBx5QSkqKevbsqVmzZvmfCy0TqqW/Q7kiLYD2hRByPN98I3Xt2rJLYmJ9i0dzLKu+hSQxsWWP9803LSqxU6dOmj59ulasWBFwQFmzZo18Pp+uvPJKjRgxQs8995zee+89zZw5U1dddZU2b97cosevq6tTdna2YmJi9NZbb+nxxx/Xbbfd1mi/E044QStWrND777+vJUuW6Mknn9RDDz0kSZoyZYp++ctf6vTTT1dpaalKS0s1ZcqURo9RXV2trKwsde/eXW+//bbWrFmjl156STfeeGPAfkVFRfr4449VVFSkP/7xj1qxYkWjEIZjC1XfjVAtVQ6gHbJlLV4HOdYyxE0u915VZVn18SH8l6qqFr+uHTt2WJKsoqIi/7bMzEzryiuvbHL/H/3oR9Yvf/lL//Uf/vCHVl5env96v379rIceesiyLMt68cUXrU6dOln79+/33/7CCy9Ykqy1a9c2W9P9999vjRgxwn994cKF1rBhwxrtd+TjPPHEE1b37t2tqiNe+3PPPWdFRUVZZWVllmVZVm5urtWvX7+ApdB/+tOfWlOmTGm2libf2w4u1Et/271UOQDzjnUMbQ2G6B5PfLx0xKmAY3r1VenSS4+/3/PPS+ed17LnbqFTTz1V55xzjp566imNHTtWu3btUnFxse655x75fD7de++9+tvf/qb9+/ertrZWNTU1Le7zsWPHDqWlpalv377+bWPGjGm03+rVq/Xwww/r448/VlVVlb7//nslJCS0+DU0PNewYcPUpUsX/7Zzzz1XdXV12rlzp5KSkiRJp59+esAv6ZSUFL377rtBPVdHF+q+G8znAeB4CCHH43JJRxwQj+miiySPp/6UTFP9OVyu+tsvukgKwRfxNddco9mzZ2vp0qVavny5TjrpJP3whz/Ub3/7Wy1ZskT5+fkaMmSIunTpojlz5qi2tta25960aZOuuOIKLVq0SFlZWUpMTNTTTz+t3/3ud7Y9x5Gio6MDrrtcLtXV1YXkudqrcKzFwnweAI6FPiF2crul/5wL19Gd8hqu5+eHJIBI0s9+9jNFRUVp5cqV+tOf/qSf//zncrlcev311zVx4kRdeeWVGjZsmAYMGKAPP/ywxY87ePBglZSUBAypfPPNNwP2eeONN9SvXz/deeedGjlypAYOHKg9e/YE7BMTE3PcTo6DBw/W9u3bVV1d7d/2+uuvKyoqSoMGDWpxzTg++m4AMI0QYrfsbKmgQDpqbQt5PPXbQ7i2RdeuXTVlyhTNmzdPpaWluvrqqyVJAwcO1IYNG/TGG29ox44d+sUvfqHy8vIWP+748eN1yimnKDc3V9u3b1dxcbHuvPPOgH0GDhyovXv36umnn9bHH3+shx9+WGvXrg3YJz09Xbt379a2bdt08OBB1dTUNHquK664Qp07d1Zubq7ee+89FRUVafbs2brqqqv8p2JgH9ZiAWASISQUsrOlTz+VioqklSvr/9y9O6QBpME111yjr776SllZWf4+HPPnz9dZZ52lrKwsjR07VsnJyZo0aVKLHzMqKkpr167Vt99+q9GjR+vaa6/Vb37zm4B9fvKTn+jmm2/WjTfeqOHDh+uNN97QggULAvaZPHmyLr74Yo0bN069e/ducphwfHy8XnzxRX355ZcaNWqUcnJydMEFF+jRRx8N/h8DLZKdna1PP/1URUVFWrlypYqKirR7924CCICQc1lNnQxuxyorK5WYmKiKiopGnSa/++477d69W/3791fnzp0NVYhQ4L0FgLY71jG0NWgJAQAARhBCAACAEYQQAABgBCEEAAAYQQhpQgfrq9sh8J4CQOQhhByhYRbOb1q4cByco+E9PXqmVQCAOUzbfgS3261u3brpwIEDkurnrGhuOXI4g2VZ+uabb3TgwAF169aN2T8BIIIQQo6SnJwsSf4ggvahW7du/vcWABAZCCFHcblcSklJUZ8+fXT48GHT5cAG0dHRtIAAQAQihDTD7XZz4ELE8fl8Ki4uVmlpqVJSUpSZmcn/UwCORQgBHMLr9SovL0/79u3zb/N4PFqyZAnrvABwJEbHAA7g9XqVk5MTEEAkaf/+/crJyZHX6zVUGQC0HiEEiHA+n095eXlNznXSsG3OnDny+XzhLg0A2oQQAkS44uLiRi0gR7IsSyUlJSouLg5jVQDQdoQQIMKVlpbauh8ARApCCBDhUlJSbN0PACIFIQSIcJmZmfJ4PM3O3utyuZSWlqbMzMwwVwYAbUMIASKc2+3WkiVLJKlREGm4np+fz3whAByHEAI4QHZ2tgoKCpSamhqw3ePxqKCggHlCADiSy+pga5xXVlYqMTFRFRUVSkhIMF0ODHLi7KNOrBlA+2H3MZQZU9EhOXX2UbfbrbFjx5ouAwBswekYdDjMPgoAkYEQgg6F2UcBIHIQQtChMPsoAEQOQgg6FGYfBYDIQcdUdCjhmn2UUSwAcHy0hKBDCcfso16vV+np6Ro3bpymTZumcePGKT09nQ6vAHAUQgg6lFDPPsrIGwBoOUIIOpxQzT7KyBsACA4zpqLDsrvfxsaNGzVu3Ljj7ldUVMSEYwAciRlTAZvYPfsoI28AIDicjgFsEq6RNwDQXhBCAJuEY+QNALQnhBDAJqEeeQMA7Q0hBLBRqEbeAEB7xOgYIASYMRVAe8ToGHQ4Tjyg2z3yBgDaI0IIIprX61VeXl7ADKQej0dLlizh1AYAOBx9QhCxmAIdANo3QggiElOgA0D7RwhBRCouLm7UAnIky7JUUlKi4uLiMFYFALATIQQRiSnQAaD9Mx5Cli5dqvT0dHXu3FkZGRnavHlzs/sePnxY99xzj0466SR17txZw4YN0/r168NYLcKFKdABoP0zGkJWr16tuXPnauHChdq6dauGDRumrKwsHThwoMn958+frz/84Q965JFH9P777+v666/XZZddpn/9619hrhyhxhToAND+GZ2sLCMjQ6NGjdKjjz4qSaqrq1NaWppmz56t22+/vdH+ffv21Z133qlZs2b5t02ePFlxcXH6y1/+0qLnZLIy52gYHSMpoINqQzBhBlIACC+7j6HGWkJqa2u1ZcsWjR8//r/FREVp/Pjx2rRpU5P3qampUefOnQO2xcXF6bXXXmv2eWpqalRZWRlwgTMwBToAtG/GJis7ePCgfD6fkpKSArYnJSXpgw8+aPI+WVlZevDBB3XeeefppJNOUmFhobxe7zGHaS5evFiLFi2ytXaET3Z2tiZOnOi4GVMBAMdnvGNqMJYsWaKBAwfq1FNPVUxMjG688UbNmDFDUVHNv4x58+apoqLCfykpKQljxbBDwxToU6dO1dixYwkgANBOGAshvXr1ktvtVnl5ecD28vJyJScnN3mf3r17a926daqurtaePXv0wQcfqGvXrhowYECzzxMbG6uEhISACwAAMM9YCImJidGIESNUWFjo31ZXV6fCwkKNGTPmmPft3LmzUlNT9f333+uZZ57RxIkTQ10uAACwmdEF7ObOnavc3FyNHDlSo0ePVn5+vqqrqzVjxgxJ0vTp05WamqrFixdLkt566y3t379fw4cP1/79+3X33Xerrq5Ot956q8mXAQAAWsFoCJkyZYo+//xz3XXXXSorK9Pw4cO1fv16f2fVvXv3BvT3+O677zR//nx98skn6tq1qy699FL9+c9/Vrdu3Qy9AgAA0FpG5wkxgXlCAABonXYzTwgAAOjYCCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADCik+kC0H74fD4VFxertLRUKSkpyszMlNvtNl0WACBCEUJgC6/Xq7y8PO3bt8+/zePxaMmSJcrOzjZYGQAgUnE6Bm3m9XqVk5MTEEAkaf/+/crJyZHX6zVUGQAgkhFC0CY+n095eXmyLKvRbQ3b5syZI5/PF+7SAAARjhCCNikuLm7UAnIky7JUUlKi4uLiMFYFAHACQgjapLS01Nb9AAAdByEEbZKSkmLrfgCAjoMQgjbJzMyUx+ORy+Vq8naXy6W0tDRlZmaGuTIAQKQjhKBN3G63lixZIkmNgkjD9fz8fOYLAQA0QghBm2VnZ6ugoECpqakB2z0ejwoKCpgnBADQJJfV1NjKdqyyslKJiYmqqKhQQkKC6XLaFWZMBYD2ze5jKDOmwjZut1tjx441XQYAwCE4HQMAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACOYJAQCEj88nFRdLpaVSSoqUmSkxqWGHRQgBAISH1yvl5Un79v13m8cjLVkisbxDh8TpGABA6Hm9Uk5OYACRpP3767d7vWbqglGEEABwMp9P2rhRWrWq/k+fz3RFjfl89S0gTS1V1rBtzpzIrB0hRQgBAKfyeqX0dGncOGnatPo/09Mjr1WhuLhxC8iRLEsqKanfLxI5Ieg5FCEEAJzISac3Skvt3S+cnBL0HIoQAgBO47TTGykp9u4XLk4Keg5FCAEAp3Ha6Y3MzPpRMC5X07e7XFJaWv1+kcJpQc+hCCEA4DROO73hdtcPw5UaB5GG6/n5kTVfiNOCnkMRQgDAaZx4eiM7WyookFJTA7d7PPXbI22eEKcFPYdisjLASZhtEtJ/T2/s39/06QKXq/72SDq9IdUHjYkTnfF/2IlBz4EIIYBTMNskGjSc3sjJqQ8cRwaRSD290cDtlsaONV3F8Tk16DkMp2MAJ6CXPo7mtNMbTuPEfiwO5LKspiJe+1VZWanExERVVFQoISHBdDnA8fl89fMSNNdJruEX2e7dfCF2RJyiC62mWiDT0uoDSAcMenYfQwkhQKTbuLF+gqTjKSpqfTM3BzKgeXw+/Ow+htInBIh0oe6lT18T4Nic0o/FgQghQKQLZS/9hr4mRzeINvQ1idS+BaH8ZcqvXiBsOB0DhIKdB7KGPiHH66UfbJ8Qp/Y1CWXLDa1CwDHZfQxldAxgN7sXvApVL30nzggZylFCjEBCe+GgVX8JIYCdQnUgC8VwTKfNCBnKtTxYJwTthcNW/SWEAHYJ9YEsO1v69NP6UTArV9b/uXt3608TOG1GyFC23DixVQg4mgNb8wghgF3CcSBr6KU/dWr9n23pq+G0lU1D2XLjtFYh4GgObc0jhCDyOeX8ptMOZE6bETKULTdOaxVCY075nggVh7bmEUIQ2Zx0ftOJBzInTf0dypYbp7UKIZCTvidCxWk/gv6DEILI5bTzm049kNnd1yRUQtlyE45WoY7+Sz1UnPY9ESpO/BEkSVYHU1FRYUmyKioqTJeCY/n+e8vyeCyrvhGx8cXlsqy0tPr9Iskzz9TX5nI1rtflqr8dbfPMM43/b6Sl2fNvG6rHbupxPR7+P7SVU78nQqHh3+Lo7x6b/y3sPoYyWRkik5PXS2HBq9Bz0oypzc1K29DCEmmnvZwkHN8TTtLwf00K/P9m4/811o5B5LLzy9vJ66VkZ0sTJzL1dyiFci0POx/7eCMWXK76EQsTJ/L/ozUc2g8iZBr6eDX13RahP4IIIbCH3Qd1p6+XwoJXkIIbscD/l+A5tR9EKDnsRxCnY9B2oWhuZr0UtAerVtWP1jielSvr536JJE5YyC9U3xNoFmvHILKEaoIc1ktBe+DUX+pOGfLqtLlu0AghBG0TyoM666XA6Zw4bNtpQ16dNNcNGqFPCNom1Ad1u89vOvWXKZyp4Zd6Tk594GhqxEIk/VJ3akdah/WDwH8RQiKZE87JhuOgbmcnz4Zfpsc7hxxJv0zhbE4aseDkjrR0BnckQkik8npl5eXJdcQXguXxyGXDEFKfz6fi4mKVlpYqJSVFmZmZcrc23DjtoO60X6ZoH5zyS53TlQgz+oREIq9X1uTJso76RWLt2ydr8uQ2nZP1er1KT0/XuHHjNG3aNI0bN07p6enytvYxndgxjHPIMMHOFZBDhdOVCDOG6EYan0/fJCWp8xdfNJkQ6yR917On4svLg/4S83q9ysnJ0dFvues/YaGgoEDZrT0AO3GWUCec7nI6/o2dhSGvOA67j6GEkAjjKyyUe/z44+/30ktyX3BByx/X51N6err2NXO+1+VyyePxaPfu3a0/NcMBB0cK5ay0CJ0wTP0N52KekHZu58aNtu7XoLi4uNkAIkmWZamkpETFbZkfwwnNzQgPpw3zxH9xuhJhRAiJMC3t7hVst7DSFnYka+l+QLNCNYEdwic7W/r00/qF31aurP9z924CCGxHCIkw7rFjVaL6vh9NqZO09z/7BSOlhR3JWrof0CxmpW0faNlEGBBCIkzm2LG6p2dPSY2DSMP1X/fsqcwgQ0hmZqY8Ho+/E+rRXC6X0tLSlBkpQ2nhXAzzBNBChJAI43a7dckTT+inkvYfdds+ST+VdPETTwTdedTtdmvJf4bSHh1EGq7n5+e3vlMq0IBhngBaiBASgbKzs3XFM8/oB6mpGitpqqSxks7zeHTFM8+0ehhtdna2CgoKlHpUhzOPx9O24bnAkZy4XgoAIxiiG8Fsndk0DI8L+DHME2iXmCekjZwUQgBHc+IEdgCOye5jKGvHAAgNp6yXAsAYQgiA0GFlUwDHYLxj6tKlS5Wenq7OnTsrIyNDmzdvPub++fn5GjRokOLi4pSWlqabb75Z3333XZiqBQAAdjEaQlavXq25c+dq4cKF2rp1q4YNG6asrCwdOHCgyf1Xrlyp22+/XQsXLtSOHTu0bNkyrV69WnfccUeYKwcAAG1ltGNqRkaGRo0apUcffVSSVFdXp7S0NM2ePVu33357o/1vvPFG7dixQ4WFhf5tv/zlL/XWW2/ptddea9Fz0jEVABARHLjoZ7tZwK62tlZbtmzR+CNWjI2KitL48eO1adOmJu9zzjnnaMuWLf5TNp988omef/55XXrppc0+T01NjSorKwMuAAAY5fVK6enSuHHStGn1f6and7jFHY2FkIMHD8rn8ykpKSlge1JSksrKypq8z7Rp03TPPffoBz/4gaKjo3XSSSdp7Nixxzwds3jxYiUmJvovaWlptr4OAACCwirTfsY7pgZj48aNuvfee/X73/9eW7duldfr1XPPPadf/epXzd5n3rx5qqio8F9KSkrCWDEAAEdglekAxobo9urVS263W+Xl5QHby8vLlZyc3OR9FixYoKuuukrXXnutJGnIkCGqrq7WzJkzdeeddyoqqnGmio2NVWxsrP0vAACAYAWzynQHGN5urCUkJiZGI0aMCOhkWldXp8LCQo0ZM6bJ+3zzzTeNgkbDdOMdbOJXAIATscp0AKOTlc2dO1e5ubkaOXKkRo8erfz8fFVXV2vGjBmSpOnTpys1NVWLFy+WJE2YMEEPPvigzjzzTGVkZGjXrl1asGCBJkyYwNonAIDIxyrTAYyGkClTpujzzz/XXXfdpbKyMg0fPlzr16/3d1bdu3dvQMvH/Pnz5XK5NH/+fO3fv1+9e/fWhAkT9Jvf/MbUSwAAoOUaVpnev7/pfiEuV/3tHWSVaRawAwAgnBy8ynS7mScEAIAOKTu7PmikpgZu93giOoCEAgvYdUQOnKUPANoVVpmWRAjpeLze+jHqRw4R83ikJUs6VPoGAONYZZrTMR0Ks/QBACIIIaSjYJY+AECEIYR0FMHM0gcAQBgQQjoKZukDAEQYQkhHwSx9AIAIQwjpKBpm6WuYDOdoLpeUltZhZukDAJhHCOko3O76YbhS4yDScD0/v8ONUQcAmBN0CElPT9c999yjvXv3hqIehBKz9AEAIkjQIWTOnDnyer0aMGCALrzwQj399NOqqakJRW0Ihexs6dNPpaIiaeXK+j937yaAAADCrtUL2G3dulUrVqzQqlWr5PP5NG3aNP385z/XWWedZXeNtmIBOwAAWsfuY2ibV9E9fPiwfv/73+u2227T4cOHNWTIEN10002aMWOGXM11gjSIEAIAQOvYfQxt9doxhw8f1tq1a7V8+XJt2LBBZ599tq655hrt27dPd9xxh1566SWtXLmyzQUCAID2KegQsnXrVi1fvlyrVq1SVFSUpk+froceekinnnqqf5/LLrtMo0aNsrVQAADQvgQdQkaNGqULL7xQjz32mCZNmqTo6OhG+/Tv31+XX365LQUCAID2KegQ8sknn6hfv37H3KdLly5avnx5q4sCAADtX9BDdA8cOKC33nqr0fa33npL//znP20pCgAAtH9Bh5BZs2appKSk0fb9+/dr1qxZthQFAADav6BDyPvvv9/kXCBnnnmm3n//fVuKAgAA7V/QISQ2Nlbl5eWNtpeWlqpTp1aP+AUAAB1M0CHkoosu0rx581RRUeHfdujQId1xxx268MILbS0OAAC0X0E3XTzwwAM677zz1K9fP5155pmSpG3btikpKUl//vOfbS8QAAC0T0GHkNTUVL3zzjv661//qu3btysuLk4zZszQ1KlTm5wzBAAAoCmt6sTRpUsXzZw50+5aAABAB9LqnqTvv/++9u7dq9ra2oDtP/nJT9pcFAAAaP9aNWPqZZddpnfffVcul0sNi/A2rJjr8/nsrRAAALRLQY+OycvLU//+/XXgwAHFx8fr3//+t1599VWNHDlSGzduDEGJAACgPQq6JWTTpk16+eWX1atXL0VFRSkqKko/+MEPtHjxYt10003617/+FYo6AQBAOxN0S4jP59MJJ5wgSerVq5c+++wzSVK/fv20c+dOe6sDAADtVtAtIWeccYa2b9+u/v37KyMjQ/fdd59iYmL0xBNPaMCAAaGoEQAAtENBh5D58+erurpaknTPPffoxz/+sTIzM9WzZ0+tXr3a9gIBAED75LIahre0wZdffqnu3bv7R8hEssrKSiUmJqqiokIJCQmmywEAwDHsPoYG1Sfk8OHD6tSpk957772A7T169HBEAAEAAJEjqBASHR2tE088kblAAABAmwU9OubOO+/UHXfcoS+//DIU9QAAgA4i6I6pjz76qHbt2qW+ffuqX79+6tKlS8DtW7duta04AADQfgUdQiZNmhSCMhzM55OKi6XSUiklRcrMlNxu01UBABDxbBkd4yS29uz1eqW8PGnfvv9u83ikJUuk7Oy2PTYAABHG6OgYHMHrlXJyAgOIJO3fX7/d6zVTFwAADhF0CImKipLb7W720iH4fPUtIE01IjVsmzOnfj8AANCkoPuErF27NuD64cOH9a9//Ut//OMftWjRItsKi2jFxY1bQI5kWVJJSf1+Y8eGrSwAAJwk6BAyceLERttycnJ0+umna/Xq1brmmmtsKSyilZbaux8AAB2QbX1Czj77bBUWFtr1cJEtJcXe/QAA6IBsCSHffvutHn74YaWmptrxcJEvM7N+FExzU9W7XFJaWv1+AACgSUGfjjl6oTrLsvT1118rPj5ef/nLX2wtLmK53fXDcHNy6gPHkR1UG/5t8vOZLwQAgGMIOoQ89NBDASEkKipKvXv3VkZGhrp3725rcREtO1sqKGh6npD8fOYJAQDgOJisrK2YMRUA0EHYfQwNuiVk+fLl6tq1q376058GbF+zZo2++eYb5ebmtrkoR3G7GYYLAEArBN0xdfHixerVq1ej7X369NG9995rS1EAAKD9CzqE7N27V/3792+0vV+/ftq7d68tRQEAgPYv6BDSp08fvfPOO422b9++XT179rSlKAAA0P4FHUKmTp2qm266SUVFRfL5fPL5fHr55ZeVl5enyy+/PBQ1AgCAdijojqm/+tWv9Omnn+qCCy5Qp071d6+rq9P06dPpEwIAAFqs1UN0P/roI23btk1xcXEaMmSI+vXrZ3dtIWH7EF0AADoI40N0GwwcOFADBw5scwEAAKBjCrpPyOTJk/Xb3/620fb77ruv0dwhAAAAzQk6hLz66qu69NJLG22/5JJL9Oqrr9pSFAAAaP+CDiFVVVWKiYlptD06OlqVlZW2FAUAANq/oEPIkCFDtHr16kbbn376aZ122mm2FAUAANq/oDumLliwQNnZ2fr44491/vnnS5IKCwu1cuVKFRQU2F4gAABon4IOIRMmTNC6det07733qqCgQHFxcRo2bJhefvll9ejRIxQ1AgCAdqjV84Q0qKys1KpVq7Rs2TJt2bJFPp/PrtpCgnlCAABoHbuPoUH3CWnw6quvKjc3V3379tXvfvc7nX/++XrzzTfbXBAAAOgYgjodU1ZWphUrVmjZsmWqrKzUz372M9XU1GjdunV0SgUAAEFpcUvIhAkTNGjQIL3zzjvKz8/XZ599pkceeSSUtQEAgHasxS0hL7zwgm666SbdcMMNTNcOAADarMUtIa+99pq+/vprjRgxQhkZGXr00Ud18ODBUNYGAADasRaHkLPPPltPPvmkSktL9Ytf/EJPP/20+vbtq7q6Om3YsEFff/11KOsEAADtTJuG6O7cuVPLli3Tn//8Zx06dEgXXnihnn32WTvrsx1DdAEAaJ2IGaIrSYMGDdJ9992nffv2adWqVW0uBgAAdBxtnqzMaWgJAQCgdSKqJQQAAKC1CCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjIiIELJ06VKlp6erc+fOysjI0ObNm5vdd+zYsXK5XI0uP/rRj8JYMQAAaCvjIWT16tWaO3euFi5cqK1bt2rYsGHKysrSgQMHmtzf6/WqtLTUf3nvvffkdrv105/+NMyVAwCAtjAeQh588EFdd911mjFjhk477TQ9/vjjio+P11NPPdXk/j169FBycrL/smHDBsXHxxNCAABwGKMhpLa2Vlu2bNH48eP926KiojR+/Hht2rSpRY+xbNkyXX755erSpUuTt9fU1KiysjLgAgAAzDMaQg4ePCifz6ekpKSA7UlJSSorKzvu/Tdv3qz33ntP1157bbP7LF68WImJif5LWlpam+sGAABtZ/x0TFssW7ZMQ4YM0ejRo5vdZ968eaqoqPBfSkpKwlghAABoTieTT96rVy+53W6Vl5cHbC8vL1dycvIx71tdXa2nn35a99xzzzH3i42NVWxsbJtrBQAA9jLaEhITE6MRI0aosLDQv62urk6FhYUaM2bMMe+7Zs0a1dTU6Morrwx1mQAAIASMtoRI0ty5c5Wbm6uRI0dq9OjRys/PV3V1tWbMmCFJmj59ulJTU7V48eKA+y1btkyTJk1Sz549TZQNAADayHgImTJlij7//HPdddddKisr0/Dhw7V+/Xp/Z9W9e/cqKiqwwWbnzp167bXX9I9//MNEyQAAwAYuy7Is00WEU2VlpRITE1VRUaGEhATT5QAA4Bh2H0MdPToGAAA4FyEEAAAYQQgBAABGEEIAAIARhBAAAGAEIQQAABhBCAEAAEYQQgAAgBGEEAAAYITxadudzufzqbi4WKWlpUpJSVFmZqbcbrfpsgAAiHiEkDbwer3Ky8vTvn37/Ns8Ho+WLFmi7Oxsg5UBABD5OB3TSl6vVzk5OQEBRJL279+vnJwceb1eQ5UBAOAMhJBW8Pl8ysvLU1Nr/zVsmzNnjnw+X7hLAwDAMQghrVBcXNyoBeRIlmWppKRExcXFYawKAABnIYS0Qmlpqa37AQDQERFCWiElJcXW/QAA6IgIIa2QmZkpj8cjl8vV5O0ul0tpaWnKzMwMc2UAADgHIaQV3G63lixZIkmNgkjD9fz8fOYLAQDgGAghrZSdna2CggKlpqYGbPd4PCooKGCeEAAAjsNlNTXOtB2rrKxUYmKiKioqlJCQ0ObHY8ZUAEBHYfcxlBlT28jtdmvs2LGmywAAwHE4HQMAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADCCEAIAAIwwHkKWLl2q9PR0de7cWRkZGdq8efMx9z906JBmzZqllJQUxcbG6pRTTtHzzz8fpmoBAIBdOpl88tWrV2vu3Ll6/PHHlZGRofz8fGVlZWnnzp3q06dPo/1ra2t14YUXqk+fPiooKFBqaqr27Nmjbt26hb94AADQJi7LsixTT56RkaFRo0bp0UcflSTV1dUpLS1Ns2fP1u23395o/8cff1z333+/PvjgA0VHR7foOWpqalRTU+O/XllZqbS0NFVUVCghIcGeFwIAQAdQWVmpxMRE246hxk7H1NbWasuWLRo/fvx/i4mK0vjx47Vp06Ym7/Pss89qzJgxmjVrlpKSknTGGWfo3nvvlc/na/Z5Fi9erMTERP8lLS3N9tcCAACCZyyEHDx4UD6fT0lJSQHbk5KSVFZW1uR9PvnkExUUFMjn8+n555/XggUL9Lvf/U6//vWvm32eefPmqaKiwn8pKSmx9XUAAIDWMdonJFh1dXXq06ePnnjiCbndbo0YMUL79+/X/fffr4ULFzZ5n9jYWMXGxoa5UgAAcDzGQkivXr3kdrtVXl4esL28vFzJyclN3iclJUXR0dFyu93+bYMHD1ZZWZlqa2sVExMT0poBAIB9jJ2OiYmJ0YgRI1RYWOjfVldXp8LCQo0ZM6bJ+5x77rnatWuX6urq/Ns+/PBDpaSkEEAAAHAYo/OEzJ07V08++aT++Mc/aseOHbrhhhtUXV2tGTNmSJKmT5+uefPm+fe/4YYb9OWXXyovL08ffvihnnvuOd17772aNWuWqZcAAABayWifkClTpujzzz/XXXfdpbKyMg0fPlzr16/3d1bdu3evoqL+m5PS0tL04osv6uabb9bQoUOVmpqqvLw83XbbbaZeAgAAaCWj84SYYPcYZwAAOop2M08IAADo2AghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADCCEAIAAIwghAAAACMiIoQsXbpU6enp6ty5szIyMrR58+Zm912xYoVcLlfApXPnzmGsFgAA2MF4CFm9erXmzp2rhQsXauvWrRo2bJiysrJ04MCBZu+TkJCg0tJS/2XPnj1hrBgAANjBeAh58MEHdd1112nGjBk67bTT9Pjjjys+Pl5PPfVUs/dxuVxKTk72X5KSksJYMQAAsEMnk09eW1urLVu2aN68ef5tUVFRGj9+vDZt2tTs/aqqqtSvXz/V1dXprLPO0r333qvTTz+9yX1rampUU1Pjv15RUSFJqqystOlVAADQMTQcOy3LsuXxjIaQgwcPyufzNWrJSEpK0gcffNDkfQYNGqSnnnpKQ4cOVUVFhR544AGdc845+ve//y2Px9No/8WLF2vRokWNtqelpdnzIgAA6GC++OILJSYmtvlxjIaQ1hgzZozGjBnjv37OOedo8ODB+sMf/qBf/epXjfafN2+e5s6d679eV1enL7/8Uj179pTL5bKlpsrKSqWlpamkpEQJCQm2PCbCg/fOuXjvnIv3zrkqKip04oknqkePHrY8ntEQ0qtXL7ndbpWXlwdsLy8vV3JycoseIzo6WmeeeaZ27drV5O2xsbGKjY0N2NatW7dW1Xs8CQkJfKAcivfOuXjvnIv3zrmiouzpUmq0Y2pMTIxGjBihwsJC/7a6ujoVFhYGtHYci8/n07vvvquUlJRQlQkAAELA+OmYuXPnKjc3VyNHjtTo0aOVn5+v6upqzZgxQ5I0ffp0paamavHixZKke+65R2effbZOPvlkHTp0SPfff7/27Nmja6+91uTLAAAAQTIeQqZMmaLPP/9cd911l8rKyjR8+HCtX7/e31l17969Ac0+X331la677jqVlZWpe/fuGjFihN544w2ddtpppl6CYmNjtXDhwkanfRD5eO+ci/fOuXjvnMvu985l2TXOBgAAIAjGJysDAAAdEyEEAAAYQQgBAABGEEIAAIARhBAbLF26VOnp6ercubMyMjK0efNm0yXhOO6++265XK6Ay6mnnmq6LDTh1Vdf1YQJE9S3b1+5XC6tW7cu4HbLsnTXXXcpJSVFcXFxGj9+vD766CMzxSLA8d67q6++utHn8OKLLzZTLAIsXrxYo0aN0gknnKA+ffpo0qRJ2rlzZ8A+3333nWbNmqWePXuqa9eumjx5cqPJR4+HENJGq1ev1ty5c7Vw4UJt3bpVw4YNU1ZWlg4cOGC6NBzH6aefrtLSUv/ltddeM10SmlBdXa1hw4Zp6dKlTd5+33336eGHH9bjjz+ut956S126dFFWVpa+++67MFeKox3vvZOkiy++OOBzuGrVqjBWiOa88sormjVrlt58801t2LBBhw8f1kUXXaTq6mr/PjfffLP+7//+T2vWrNErr7yizz77TNnZ2cE9kYU2GT16tDVr1iz/dZ/PZ/Xt29davHixwapwPAsXLrSGDRtmugwESZK1du1a//W6ujorOTnZuv/++/3bDh06ZMXGxlqrVq0yUCGac/R7Z1mWlZuba02cONFIPQjOgQMHLEnWK6+8YllW/ecsOjraWrNmjX+fHTt2WJKsTZs2tfhxaQlpg9raWm3ZskXjx4/3b4uKitL48eO1adMmg5WhJT766CP17dtXAwYM0BVXXKG9e/eaLglB2r17t8rKygI+g4mJicrIyOAz6BAbN25Unz59NGjQIN1www364osvTJeEJlRUVEiSf+G6LVu26PDhwwGfvVNPPVUnnnhiUJ89QkgbHDx4UD6fzz+7a4OkpCSVlZUZqgotkZGRoRUrVmj9+vV67LHHtHv3bmVmZurrr782XRqC0PA54zPoTBdffLH+9Kc/qbCwUL/97W/1yiuv6JJLLpHP5zNdGo5QV1enOXPm6Nxzz9UZZ5whqf6zFxMT02hB2GA/e8anbQdMuOSSS/x/Hzp0qDIyMtSvXz/97W9/0zXXXGOwMqDjuPzyy/1/HzJkiIYOHaqTTjpJGzdu1AUXXGCwMhxp1qxZeu+990LSb46WkDbo1auX3G53o97A5eXlSk5ONlQVWqNbt2465ZRTtGvXLtOlIAgNnzM+g+3DgAED1KtXLz6HEeTGG2/U//t//09FRUXyeDz+7cnJyaqtrdWhQ4cC9g/2s0cIaYOYmBiNGDFChYWF/m11dXUqLCzUmDFjDFaGYFVVVenjjz9WSkqK6VIQhP79+ys5OTngM1hZWam33nqLz6AD7du3T1988QWfwwhgWZZuvPFGrV27Vi+//LL69+8fcPuIESMUHR0d8NnbuXOn9u7dG9Rnj9MxbTR37lzl5uZq5MiRGj16tPLz81VdXa0ZM2aYLg3HcMstt2jChAnq16+fPvvsMy1cuFBut1tTp041XRqOUlVVFfDLePfu3dq2bZt69OihE088UXPmzNGvf/1rDRw4UP3799eCBQvUt29fTZo0yVzRkHTs965Hjx5atGiRJk+erOTkZH388ce69dZbdfLJJysrK8tg1ZDqT8GsXLlSf//733XCCSf4+3kkJiYqLi5OiYmJuuaaazR37lz16NFDCQkJmj17tsaMGaOzzz675U9k9zCejuiRRx6xTjzxRCsmJsYaPXq09eabb5ouCccxZcoUKyUlxYqJibFSU1OtKVOmWLt27TJdFppQVFRkSWp0yc3NtSyrfpjuggULrKSkJCs2Nta64IILrJ07d5otGpZlHfu9++abb6yLLrrI6t27txUdHW3169fPuu6666yysjLTZcOymnzfJFnLly/37/Ptt99a//M//2N1797dio+Pty677DKrtLQ0qOdx/efJAAAAwoo+IQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEA2gWXy6V169aZLgNAEAghANrs6quvlsvlanS5+OKLTZcGIIKxgB0AW1x88cVavnx5wLbY2FhD1QBwAlpCANgiNjZWycnJAZfu3btLqj9V8thjj+mSSy5RXFycBgwYoIKCgoD7v/vuuzr//PMVFxennj17aubMmaqqqgrY56mnntLpp5+u2NhYpaSk6MYbbwy4/eDBg7rssssUHx+vgQMH6tlnnw3tiwbQJoQQAGGxYMECTZ48Wdu3b9cVV1yhyy+/XDt27JAkVVdXKysrS927d9fbb7+tNWvW6KWXXgoIGY899phmzZqlmTNn6t1339Wzzz6rk08+OeA5Fi1apJ/97Gd65513dOmll+qKK67Ql19+GdbXCSAItq79C6BDys3Ntdxut9WlS5eAy29+8xvLsuqXBb/++usD7pORkWHdcMMNlmVZ1hNPPGF1797dqqqq8t/+3HPPWVFRUf6l3fv27WvdeeedzdYgyZo/f77/elVVlSXJeuGFF2x7nQDsRZ8QALYYN26cHnvssYBtPXr08P99zJgxAbeNGTNG27ZtkyTt2LFDw4YNU5cuXfy3n3vuuaqrq9POnTvlcrn02Wef6YILLjhmDUOHDvX/vUuXLkpISNCBAwda+5IAhBghBIAtunTp0uj0iF3i4uJatF90dHTAdZfLpbq6ulCUBMAG9AkBEBZvvvlmo+uDBw+WJA0ePFjbt29XdXW1//bXX39dUVFRGjRokE444QSlp6ersLAwrDUDCC1aQgDYoqamRmVlZQHbOnXqpF69ekmS1qxZo5EjR+oHP/iB/vrXv2rz5s1atmyZJOmKK67QwoULlZubq7vvvluff/65Zs+erauuukpJSUmSpLvvvlvXX3+9+vTpo0suuURff/21Xn/9dc2ePTu8LxSAbQghAGyxfv16paSkBGwbNGiQPvjgA0n1I1eefvpp/c///I9SUlK0atUqnXbaaZKk+Ph4vfjii8rLy9OoUaMUHx+vyZMn68EHH/Q/Vm5urr777js99NBDuuWWW9SrVy/l5OSE7wUCsJ3LsizLdBEA2jeXy6W1a9dq0qRJpksBEEHoEwIAAIwghAAAACPoEwIg5DjrC6AptIQAAAAjCCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjPj/qny8BGCItwcAAAAASUVORK5CYII=", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiEAAAISCAYAAAAa+R+EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABCRklEQVR4nO3dfVxUdf7//+cwCoIFeZGADImWqZUX5QVZy6ZlYe6aRmymleZ2sbVqkN9umZWZtaufLrawzc1P/UzbdjFXQuvzqbU1wqSy7KOr2WZuFiUSYGZCoIHOnN8fxKwjoAyc4T0Dj/vtNjecM2fOvIbxcJ7zfr/P+zgsy7IEAADQysJMFwAAANonQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwwmgI2bhxo8aPH6+ePXvK4XBo7dq1J33Ohg0bdMEFFygiIkJnnXWWVqxYEfA6AQCA/YyGkKqqKg0ePFhLlixp0vqFhYX6xS9+odGjR2vbtm3KzMzULbfcojfffDPAlQIAALs5guUCdg6HQ2vWrNHEiRMbXWfOnDl6/fXX9cknn3iXXXfddTp48KDWrVvXClUCAAC7dDBdgD82bdqkMWPG+CxLTU1VZmZmo8+prq5WdXW1977H49GBAwfUrVs3ORyOQJUKAECbY1mWfvjhB/Xs2VNhYS3vTAmpEFJaWqrY2FifZbGxsaqoqNDhw4cVGRlZ7zmLFi3SggULWqtEAADavKKiIrlcrhZvJ6RCSHPMnTtXs2fP9t4vLy/XGWecoaKiIkVHRxusDADQnrndbr3//vsqLS1VXFycLrroIjmdzmZv77XXXtONN97Y6OMvvfSSrrrqqmZvX5IqKiqUmJioU089tUXbqRNSISQuLk5lZWU+y8rKyhQdHd1gK4gkRUREKCIiot7y6OhoQggAwIjc3FxlZGRo79693mUul0uLFy9WWlqa39tzu92aO3duo487HA7dd999mjx5couCzrHbs0NIzRMycuRI5eXl+Sxbv369Ro4caagiAEBb53a7tWHDBq1cuVIbNmyQ2+1u0fZyc3OVnp7uE0Akqbi4WOnp6crNzfV7mwUFBfW2dyzLslRUVKSCggK/tx1IRkNIZWWltm3bpm3btkmqPQV327Zt2rNnj6TarpSpU6d617/99tv15Zdf6p577tFnn32mP/3pT/rb3/6mu+66y0T5AIAgYXdQqJObm6ukpCSNHj1aU6ZM0ejRo5WUlNSsoFBXZ0ZGhho6MbVuWWZmpt/1l5SU2Lpeq7EMys/PtyTVu02bNs2yLMuaNm2adckll9R7zpAhQ6zw8HCrT58+1vLly/16zfLyckuSVV5ebs+bAAAY9corr1gul8vnOOJyuaxXXnmlxdt1OBz1jlEOh8NyOBzN2n5jx73jb/n5+UGx3ePZfQwNmnlCWktFRYViYmJUXl7e6JgQy7J09OhR25I0zHI6nerQoQOnZAN+cLvdKigoUElJieLj45WSkmLLWAK7t1vXtXH8oaxuf8/JyWn2GIukpKRGuzgcDodcLpcKCwv9qn/lypWaMmXKSdfLzs7W5MmT/a63uLi4wVaW5tZ7vKYcQ/0RUgNTW0NNTY1KSkp06NAh06XARlFRUYqPj1d4eLjpUoCgZ/egyUBt92RdGw6HQ5mZmZowYYLfB15/xliMGjWqyduNj4+3db06TqdTixcvVnp6uhwOh8/vpC6QZWVl2RIk7UQIOYbH4/GmxJ49eyo8PJxvzyHOsizV1NTo22+/VWFhofr27WvLBDtAW9VYy0LdoMnmtiwEYruBCgpS4MZYpKSkyOVynbTFIiUlxa/tSlJaWppycnIaDHpZWVktCpCBQgg5Rk1NjTwejxITExUVFWW6HNgkMjJSHTt21Ndff62amhp16tTJdEmAbezs3ghUy0KgthvIwZih2mKRlpamCRMmBKQrLRD4StgAvim3PXymaIvsPnMjUKd5Bmq7gQoK0n9aLBprDXc4HEpMTGxRi0VCQoLPcpfL1eyWpmM5nU6NGjVKkydP1qhRo4I2gEiEEAAISYGYayJQLQuB7toIRFCoa7Go287x25Va3mLx1VdfKT8/X9nZ2crPz1dhYWFQdpkEEiEEAFqBnfNYBGquiUC1LAS6a0MKXFCgxSLAbDnRN4Sc6Bznw4cPW59++ql1+PDhFr/O0aNHrfz8fCs7O9vKz8+3jh492uJttrZevXpZTz31VJPXrztP/fvvvw9YTc1l52eLti0Q+67d81gEak6Io0ePWi6Xq8G5MfTT/BiJiYl+/04Ctd06Df1+ExMTWzxPyLH1h/rfc7vYPU8IIeQYdh2oAjVxTmNO9odo/vz5zdruvn37rKqqqiavX11dbZWUlFgej6dZrxdIhBA0RSD23UBMeJWdnd2kEJKdnd3seo+vuSX1BnK7dQgKrYMQ0kKBDiGB+INzMiUlJd5bVlaWFR0d7bPshx9+8K7r8XisI0eO2F5DsCOE4GQCse/WtQA0FhKa2wIQ6NkxA9WyEOgWCwQeIaSF/A0hHo/HqqysbNKtvLzcSkhIOOEfHJfLZZWXlzdpe81pUVi+fLkVExPjvV/3x+qNN96wLrjgAqtjx45Wfn6+tXv3buuqq66yevToYXXu3NkaNmyYtX79ep9tHd8dI8l6/vnnrYkTJ1qRkZHWWWedZb366qv1XquuO6aulnXr1ln9+/e3OnfubKWmplrffPON9zlHjhyxZs2aZcXExFhdu3a17rnnHmvq1KnWhAkT/H7vJ0IIwYmEWlgIdPdG3WsEomWBFovQZncIYWDqSRw6dEinnHJKk24xMTEqLi5udFuWZWnv3r2KiYlp0vbsnLX13nvv1X/9139p586dGjRokCorKzVu3Djl5eXpn//8p8aOHavx48d7Lx7YmAULFujaa6/Vxx9/rHHjxun666/XgQMHGl3/0KFDeuKJJ/TSSy9p48aN2rNnj+6++27v448++qj++te/avny5XrvvfdUUVGhtWvX2vW2gSYJ1CmkgTorJNADMuteIxCDJhmMiWMRQtqJhx9+WJdffrnOPPNMde3aVYMHD9ZvfvMbnXfeeerbt68eeeQRnXnmmXrttddOuJ2bbrpJkydP1llnnaWFCxeqsrJSmzdvbnT9I0eOaOnSpRo2bJguuOACzZw5U3l5ed7H//jHP2ru3Lm6+uqr1b9/fz3zzDM67bTT7HrbQJMEKiwEch6LQJ+5AbQGZkw9iaioKFVWVjZp3Y0bN2rcuHEnXe+NN97Qz3/+8ya9tl2GDRvmc7+yslIPPfSQXn/9dZWUlOjo0aM6fPjwSVtCBg0a5P13586dFR0drX379jW6flRUlM4880zv/fj4eO/65eXlKisr04gRI7yPO51ODR06VB6Px6/3B7REoMJCIKfolkJvdkzgeISQk3A4HOrcuXOT1r3iiiua9AfniiuuaPU/Ese/h7vvvlvr16/XE088obPOOkuRkZFKT09XTU3NCbfTsWNHn/sOh+OEgaGh9Rv63aBtCpUrsQYqLLTGRcXqujeAUER3jI1ao5/WLu+9955uuukmXX311Ro4cKDi4uL01VdftWoNMTExio2N1UcffeRd5na7tXXr1latA4Fh95TigdxuIPdduk2AxhFCbBYqf3D69u2r3Nxcbdu2Tdu3b9eUKVOMdIHMmjVLixYt0quvvqpdu3YpIyND33//PVcvDnGBmFI8kNuVArvvMkU30DC6YwIgFPppn3zySf3617/WRRddpO7du2vOnDmqqKho9TrmzJmj0tJSTZ06VU6nU7fddptSU1OD6ncF/4TalViPFch9l24ToD6H1c466CsqKhQTE6Py8nJFR0f7PPbjjz+qsLBQvXv35nLvhng8Hg0YMEDXXnutHnnkEdu2y2fbejZs2KDRo0efdL38/Hy/DsqB2i6ApjvRMbQ5aAmBUV9//bX+8Y9/6JJLLlF1dbWeeeYZFRYWasqUKaZLQzOF2pVYAZjDmBAYFRYWphUrVmj48OG6+OKLtWPHDr311lsaMGCA6dLQTKF2JVYA5tASAqMSExP13nvvmS4DNgrU6a6BnnMDQOujJQSArQJ1umsonQIPoGkIIQBsF6jTXUPlFHgATcPZMcfgDIq2i8/2xEJlZtNAbxfAiXF2DABb5ebmKiMjw2cCMJfLpcWLF7e4ZSFQc2Mw5wbQNtAdA7RjgZyBFABOhhACtFMnm4FUkjIzM+V2u1u7NADtBCEkUNxuacMGaeXK2p9B/od81KhRyszM9N5PSkpSVlbWCZ/jcDi0du3aFr+2XduBfwoKCuq1gBzLsiwVFRWpoKCgFasC0J4QQgIhN1dKSpJGj5amTKn9mZRUuzwAxo8fr7Fjxzb4WEFBgRwOhz7++GO/tvnRRx/ptttus6M8r4ceekhDhgypt7ykpERXXnmlra+Fk2MGUgCmEULslpsrpadLx3/DLC6uXR6AIHLzzTdr/fr1DX6rXb58uYYNG6ZBgwb5tc3TTz9dUVFRdpV4QnFxcYqIiGiV18J/MAMpANMIISdjWVJVVdNuFRXSnXfWPqeh7UhSRkbtek3ZXhPPnv7lL3+p008/XStWrPBZXllZqdWrV2vixImaPHmyEhISFBUVpYEDB2rlypUn3Obx3TGff/65fv7zn6tTp04655xztH79+nrPmTNnjs4++2xFRUWpT58+mjdvno4cOSJJWrFihRYsWKDt27fL4XDI4XB46z2+O2bHjh269NJLFRkZqW7duum2225TZWWl9/GbbrpJEydO1BNPPKH4+Hh169ZNM2bM8L4WmqZuBtLjJ/6q43A4lJiYyAykAAKGU3RP5tAh6ZRT7NmWZdW2kMTENG39ykqpc+eTrtahQwdNnTpVK1as0P333+89qKxevVput1s33HCDVq9erTlz5ig6Olqvv/66brzxRp155pkaMWLESbfv8XiUlpam2NhYffjhhyovL/cZP1Ln1FNP1YoVK9SzZ0/t2LFDt956q0499VTdc889mjRpkj755BOtW7dOb731liQppoHfQ1VVlVJTUzVy5Eh99NFH2rdvn2655RbNnDnTJ2Tl5+crPj5e+fn52r17tyZNmqQhQ4bo1ltvPen7Qa26GUjT09PlcDh8BqgyAymAVmG1M+Xl5ZYkq7y8vN5jhw8ftj799FPr8OHD/1lYWWlZtfGh9W+VlU1+Xzt37rQkWfn5+d5lKSkp1g033NDg+r/4xS+s//f//p/3/iWXXGJlZGR47/fq1ct66qmnLMuyrDfffNPq0KGDVVxc7H3873//uyXJWrNmTaM1Pf7449bQoUO99+fPn28NHjy43nrHbue5556zunTpYlUe895ff/11KywszCotLbUsy7KmTZtm9erVyzp69Kh3nV/96lfWpEmTGq2lwc8WlmVZ1iuvvGK5XC5LkveWmJhovfLKK6ZLAxBkTnQMbQ5aQk4mKqq2RaIpNm6Uxo07+XpvvCH9/OdNe+0m6t+/vy666CK98MILGjVqlHbv3q2CggI9/PDDcrvdWrhwof72t7+puLhYNTU1qq6ubvKYj507dyoxMVE9e/b0Lhs5cmS99VatWqWnn35aX3zxhSorK3X06FG/Z9TbuXOnBg8erM7HtABdfPHF8ng82rVrl2JjYyVJ5557rs839Pj4eO3YscOv10KttLQ0TZgwgRlIAbQ6QsjJOBxN6hKRJF1xheRy1Q5CbWg8h8NR+/gVV0gB+AN/8803a9asWVqyZImWL1+uM888U5dccokeffRRLV68WFlZWRo4cKA6d+6szMxM1dTU2PbamzZt0vXXX68FCxYoNTVVMTExevnll/WHP/zBttc4VseOHX3uOxwOeTyegLxWe8AMpABMYGCqnZxO6aerfOr4wX5197OyAhJAJOnaa69VWFiYsrOz9ec//1m//vWv5XA49N5772nChAm64YYbNHjwYPXp00f//ve/m7zdAQMGqKioyOdUzQ8++MBnnffff1+9evXS/fffr2HDhqlv3776+uuvfdYJDw8/6cRXAwYM0Pbt21VVVeVd9t577yksLEz9+vVrcs0AgOBHCLFbWpqUkyMdd5VPuVy1ywN4lc9TTjlFkyZN0ty5c1VSUqKbbrpJktS3b1+tX79e77//vnbu3Knf/OY3Kisra/J2x4wZo7PPPlvTpk3T9u3bVVBQoPvvv99nnb59+2rPnj16+eWX9cUXX+jpp5/WmjVrfNZJSkpSYWGhtm3bpv3796u6urrea11//fXq1KmTpk2bpk8++UT5+fmaNWuWbrzxRm9XTHvmdru1YcMGrVy5Uhs2bGA2UwAhjRASCGlp0ldfSfn5UnZ27c/CwoAGkDo333yzvv/+e6WmpnrHcDzwwAO64IILlJqaqlGjRikuLk4TJ05s8jbDwsK0Zs0aHT58WCNGjNAtt9yi3//+9z7rXHXVVbrrrrs0c+ZMDRkyRO+//77mzZvns84111yjsWPHavTo0Tr99NMbPE04KipKb775pg4cOKDhw4crPT1dl112mZ555hn/fxltTG5urpKSkjR69GhNmTJFo0ePVlJSEtd3ARCyHJbVxMko2ogTXYaYy723XaH+2dZdaO743bXuVNqcnJwWX/EWAE7mRMfQ5qAlBAhyXGgOQFtFCAGCHBeaA9BWEUKAIMeF5gC0VcwTAgSA2+22bfIvLjQHoK2iJaQB7WysbrvQmp+p3WexcKE5AG0VIeQYdbNwHjp0yHAlsFvdZ3r8TKt2qzuL5fgxHMXFxUpPT29WEKm70JykekGEC80BCGWconuckpISHTx4UD169FBUVFSj3z4RGizL0qFDh7Rv3z6ddtppAe2ycLvdSkpKanQQqcPhkMvlUmFhYbMCQ25urjIyMny2n5iYqKysLE7PBdAq7D5FlxByHMuyVFpaqoMHD7Z+cQiY0047TXFxcQENlRs2bNDo0aNPul5+fn6zr9Ni51gTAPCX3SGEganHcTgcio+PV48ePXTkyBHT5cAGHTt2bJUDdWucxcKF5gC0JYSQRjidTr5hwi+cxQIA/mFgKmATzmIBAP8QQgCbcBYLAPiHEALYKC0tTTk5OUpISPBZ7nK5uMgcAByHs2OAAOAsFgBtEWfHACGAs1gA4OTojgEAAEYQQgAAgBF0x6DdYtwGAJhFCEG71NB1WFwulxYvXswZLADQSuiOQbsTiCvdAgD8RwhBu+J2u5WRkaGGzkyvW5aZmSm3293apQFAu0MIQbtSUFBQrwXkWJZlqaioSAUFBa1YFQC0T4QQtCutcaVbAEDTEELQrnClWwAIHoQQtCtc6RYAggchBO0KV7oFgOBBCEG7w5VuASA4cBVdtFvMmAoA/uEquoBNuNItAJhFdwwAADCCEAIAAIwghAAAACMYE4KgxwBSAGibCCEIarm5ucrIyPC53ovL5dLixYs5lRYAQhzdMQhaubm5Sk9Pr3fBueLiYqWnpys3N9dQZQAAOxBCEJTcbrcyMjLU0DQ2dcsyMzPldrtbuzQAgE0IIQhKBQUF9VpAjmVZloqKilRQUNCKVQEA7EQIQVAqKSmxdT0AQPAhhCAoxcfH27oeACD4EEIQlFJSUuRyuepd6baOw+FQYmKiUlJSWrkyAIBdCCEISk6nU4sXL5akekGk7n5WVhbzhQBACCOEwDZut1sbNmzQypUrtWHDhhafuZKWlqacnBwlJCT4LHe5XMrJyWGeEAAIccZDyJIlS5SUlKROnTopOTlZmzdvbnTdI0eO6OGHH9aZZ56pTp06afDgwVq3bl0rVovG5ObmKikpSaNHj9aUKVM0evRoJSUltXguj7S0NH311VfKz89Xdna28vPzVVhYSAABgDbAYTU0EUMrWbVqlaZOnaqlS5cqOTlZWVlZWr16tXbt2qUePXrUW3/OnDn6y1/+oueff179+/fXm2++qdmzZ+v999/X+eef36TXrKioUExMjMrLyxUdHW33W2qX6iYVO/6/Ul23Ca0WANA22H0MNRpCkpOTNXz4cD3zzDOSJI/Ho8TERM2aNUv33ntvvfV79uyp+++/XzNmzPAuu+aaaxQZGam//OUvTXpNQoi93G63kpKSGp3Tw+FwyOVyqbCwkPEbABDi7D6GGuuOqamp0ZYtWzRmzJj/FBMWpjFjxmjTpk0NPqe6ulqdOnXyWRYZGal333230deprq5WRUWFzw32YVIxAEBzGQsh+/fvl9vtVmxsrM/y2NhYlZaWNvic1NRUPfnkk/r888/l8Xi0fv165ebmnnDCqkWLFikmJsZ7S0xMtPV9tHdMKgYAaC7jA1P9sXjxYvXt21f9+/dXeHi4Zs6cqenTpyssrPG3MXfuXJWXl3tvRUVFrVhx28ekYgCA5jIWQrp37y6n06mysjKf5WVlZYqLi2vwOaeffrrWrl2rqqoqff311/rss890yimnqE+fPo2+TkREhKKjo31usA+TigEAmstYCAkPD9fQoUOVl5fnXebxeJSXl6eRI0ee8LmdOnVSQkKCjh49qldeeUUTJkwIdLloBJOKAQCay2h3zOzZs/X888/rxRdf1M6dO3XHHXeoqqpK06dPlyRNnTpVc+fO9a7/4YcfKjc3V19++aUKCgo0duxYeTwe3XPPPabeAsSkYgCA5ulg8sUnTZqkb7/9Vg8++KBKS0s1ZMgQrVu3zjtYdc+ePT7jPX788Uc98MAD+vLLL3XKKado3Lhxeumll3TaaacZegeok5aWpgkTJqigoEAlJSWKj49XSkoKLSAAgEYZnSfEBOYJAQCgedrMPCEAAKB9I4QAAAAjCCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwooPpAtD63G63CgoKVFJSovj4eKWkpMjpdJouC0BzuN1SQYFUUiLFx0spKRL7M0IEIaSdyc3NVUZGhvbu3etd5nK5tHjxYqWlpRmsDIDfcnOljAzpmP1ZLpe0eLHE/owQQHdMO5Kbm6v09HSfACJJxcXFSk9PV25urqHKAPgtN1dKT/cNIJJUXFy7nP0ZIcBhWZZluojWVFFRoZiYGJWXlys6Otp0Oa3G7XYrKSmpXgCp43A45HK5VFhYSNcMEOzcbikpqX4AqeNw1LaIFBbSNdMeBbCLzu5jKC0h7URBQUGjAUSSLMtSUVGRCgoKWrEqAM1SUNB4AJEky5KKimrXQ/uSm1sbUEePlqZMqf2ZlBS0LWOEkHaipKTE1vXaBLdb2rBBWrmy9qfbbboioGmaup+2p/0ZIdlFx8DUdiI+Pt7W9UIeA/oQypq6nwbj/szZPP9h5+/C7a79m9bQCAvLqu2iy8yUJkwIqt83LSHtREpKilwulxwOR4OPOxwOJSYmKiUlpZUrMyAEvy0APlJSakNzI/uzHA4pMbF2vWASYl0FAWX37yJEu+gIIe2E0+nU4sWLJaleEKm7n5WV1fYHpZ7s24JU+20hWLtm6EKCVPtN9qf9uV4QqbuflRVU33gJ/8cIxO8iRLvoCCHtSFpamnJycpSQkOCz3OVyKScnp33MExKi3xYk8S2ytYRK0EtLk3JypOP2Z7lctcuDaX8m/PtuKxC/i1DtorPamfLyckuSVV5ebroUY44ePWrl5+db2dnZVn5+vnX06FHTJbWe7GzLqt3VT3zLzjZdqa9XXrEsh6N+nQ5H7e2VV0xX2Da88opluVy+v2OXK7h/v0ePWlZ+fu3/2fz82vvBJj+/aftdfr7pSuuz+/9EoH4XR4/W1tXQ34m6vxWJiS3+/2H3MZSBqe2Q0+nUqFGjTJdhRih+WwjRAWchp66J/Pjfc10TebC1LtRxOqVg359DtKsgIP8nAvW7qOuiS0+v/ZtwbM3B2kUnumPQ3oTigL5Q7kIKFaHeXRDsWiP8292NFordJqHURfcTQgiCn51/XEJxQF+ofosMJQS9wAp0+A/EeKlA/Z8I9O8iLU366ispP1/Kzq79WVgYlAFEIoQg2AXij0uofVsIxS6kQLP7W29rBL1QGfAaCIEM/4E66ybQ3SZS4L4I1XXRTZ5c+zOYvlQdz5aRJSGEgakhJNCDMUNhQJ9ltc6As1D5XVhWYAaPBnrgZCgOeA2Ehn4PiYnN/z3U7RuNfV4t2TdM/J9oye+ildh9DCWEIDgF8o9LKKoLZMcHETsCWSgdIAMVTAMZ9DizyZedgTeQQYHw3yBCSAsRQkJEKJ/SFyiB+OYUSgfIQAfTQAQ9wnRgBfqU+0CG/xBl9zGUMSEITgzGrM/uAWehdkZIoAePBmKsEANeAyvQ46VCbfxYCGKeEAQnBmM2zM45Ifw5QAbDPBStEUzT0mrnW7HromKE6cCqO9OkuLjhMO1w1D7eklPu7f4/AR+EEASn1vjj0t6F2gGytYKpnUGPMB1YrTVBVyhMCBei6I5BcArF+TyOFQqnY4baATIUJ5oLxZpDDV0mIY0QguAVqn9cQuVCc6F2gAzFYBqKNYeiEJugC//hsKyG2rrbroqKCsXExKi8vFzR0dGmy0FTuN2h0x/b2LUm6g44wRae6uqVGm7KDrZ6pdqaMzJ8x7MkJtYezIOt1jqhWDPQALuPoYSQ9iiUDuqhxO2ubfFobLBn3TiWwsLg+n2H4gEyFP8Ph2LNwHEIIS3U7kNIQwccl6u2yThYDzihYsOG2q6Xk8nPD75BbhwgATSB3cdQzo5pT0L1UuWhItTONjkWo/9xPIIpWgEDU9uLUJuYKhSF2tkmQGNCZXA1Qh4hpL1g5sbAC7WzTYCGBOqqtEADCCHBzM65JkK5qyBUcDomQh0tpmhlhJBgZXdzKF0FrSNU5zYBJFpM0eoYmBqMAjGAlGnQWw/XmkCoosUUrYyWkGATqOZQugpaV93ZJpMn1/7k94pQQIspWhkhJNgEsjmUrgIAJ8LgarQyumOCTaCbQ+kqANCY1roqLfATQkiwaY3mUCamAtCYuhbThmZWDuap/BGSmLY92LjdOhQbq07ffddgX5lH0o/duimqrIxvIwAChxlT0QCmbW/j3JIyJP23agPHsUHE89PPTEnPSuLPAYIeB7LQRYspWgEDU4NMQUGB/r/vvlO6pOLjHtsrKV3S8999pwLO00ewY+pvACdBCAkyJT8NOF0jKUnSKEmTf/rZ+6flx64HBCWm/gbQBISQIBN/zIBTj6R3JL38009PI+sBQYWpvwE0ESEkyKSkpMjlcsnRyHn6DodDiYmJSuE8fQQrpv4G0ESEkCDjdDq1+KeZTY8PInX3s7Ky5GRwH4IVU38DaCJCSBBKS0tTTk6OEo6b2dTlciknJ0dpnKePYMbU3wCaiHlCgpjb7VZBQYFKSkoUHx+vlJQUWkAQ/Nzu2rNgTnaxxMJCTtcFQgzzhLQjTqdTozhPH6GGqb8BNBHdMQDsx8USATQBLSEAAoOLJQI4CUIIgMBh6m8AJ0B3DAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADDCeAhZsmSJkpKS1KlTJyUnJ2vz5s0nXD8rK0v9+vVTZGSkEhMTddddd+nHH39spWoBAIBdjIaQVatWafbs2Zo/f762bt2qwYMHKzU1Vfv27Wtw/ezsbN17772aP3++du7cqWXLlmnVqlW67777WrlyAADQUkZDyJNPPqlbb71V06dP1znnnKOlS5cqKipKL7zwQoPrv//++7r44os1ZcoUJSUl6YorrtDkyZNP2noCAACCj7EQUlNToy1btmjMmDH/KSYsTGPGjNGmTZsafM5FF12kLVu2eEPHl19+qTfeeEPjxo1r9HWqq6tVUVHhcwMAAOYZu4ru/v375Xa7FRsb67M8NjZWn332WYPPmTJlivbv36+f/exnsixLR48e1e23337C7phFixZpwYIFttYOAABazvjAVH9s2LBBCxcu1J/+9Cdt3bpVubm5ev311/XII480+py5c+eqvLzceysqKmrFigEAQGOMtYR0795dTqdTZWVlPsvLysoUFxfX4HPmzZunG2+8UbfccoskaeDAgaqqqtJtt92m+++/X2Fh9TNVRESEIiIi7H8DAACgRYy1hISHh2vo0KHKy8vzLvN4PMrLy9PIkSMbfM6hQ4fqBQ2n0ylJsiwrcMUCAADbGWsJkaTZs2dr2rRpGjZsmEaMGKGsrCxVVVVp+vTpkqSpU6cqISFBixYtkiSNHz9eTz75pM4//3wlJydr9+7dmjdvnsaPH+8NIwAAIDQYDSGTJk3St99+qwcffFClpaUaMmSI1q1b5x2sumfPHp+WjwceeEAOh0MPPPCAiouLdfrpp2v8+PH6/e9/b+otAACAZnJY7awfo6KiQjExMSovL1d0dLTpcgAACBl2H0ND6uwYAADQdhBCAACAEYQQAABgBCEEAAAYQQgBAABGEEIAAIARhBAAAGAEIQQAABhBCAEAAEYQQgAAgBF+h5CkpCQ9/PDD2rNnTyDqAQAA7YTfISQzM1O5ubnq06ePLr/8cr388suqrq4ORG0AAKANa1YI2bZtmzZv3qwBAwZo1qxZio+P18yZM7V169ZA1AgAANqgFl9F98iRI/rTn/6kOXPm6MiRIxo4cKDuvPNOTZ8+XQ6Hw646bcNVdAEAaB67j6EdmvvEI0eOaM2aNVq+fLnWr1+vCy+8UDfffLP27t2r++67T2+99Zays7NbXCAAAGib/A4hW7du1fLly7Vy5UqFhYVp6tSpeuqpp9S/f3/vOldffbWGDx9ua6EAAKBt8TuEDB8+XJdffrmeffZZTZw4UR07dqy3Tu/evXXdddfZUiAAAGib/A4hX375pXr16nXCdTp37qzly5c3uygAAND2+X12zL59+/Thhx/WW/7hhx/q//7v/2wpCgAAtH1+h5AZM2aoqKio3vLi4mLNmDHDlqIAAEDb53cI+fTTT3XBBRfUW37++efr008/taUoAADQ9vkdQiIiIlRWVlZveUlJiTp0aPYZvwAAoJ3xO4RcccUVmjt3rsrLy73LDh48qPvuu0+XX365rcUBAIC2y++miyeeeEI///nP1atXL51//vmSpG3btik2NlYvvfSS7QUCAIC2ye8QkpCQoI8//lh//etftX37dkVGRmr69OmaPHlyg3OGAAAANKRZgzg6d+6s2267ze5aAABAO9LskaSffvqp9uzZo5qaGp/lV111VYuLAgAAbV+zZky9+uqrtWPHDjkcDtVdhLfuirlut9veCgEAQJvk99kxGRkZ6t27t/bt26eoqCj961//0saNGzVs2DBt2LAhACUCAIC2yO+WkE2bNuntt99W9+7dFRYWprCwMP3sZz/TokWLdOedd+qf//xnIOoEAABtjN8tIW63W6eeeqokqXv37vrmm28kSb169dKuXbvsrQ4AALRZfreEnHfeedq+fbt69+6t5ORkPfbYYwoPD9dzzz2nPn36BKJGAADQBvkdQh544AFVVVVJkh5++GH98pe/VEpKirp166ZVq1bZXiAAAGibHFbd6S0tcODAAXXp0sV7hkwwq6ioUExMjMrLyxUdHW26HAAAQobdx1C/xoQcOXJEHTp00CeffOKzvGvXriERQAAAQPDwK4R07NhRZ5xxBnOBAACAFvP77Jj7779f9913nw4cOBCIegAAQDvh98DUZ555Rrt371bPnj3Vq1cvde7c2efxrVu32lYcAABou/wOIRMnTgxAGQAAoL2x5eyYUMLZMQAANI/Rs2MAAADs4nd3TFhY2AlPx+XMGQAA0BR+h5A1a9b43D9y5Ij++c9/6sUXX9SCBQtsKwwAALRtto0Jyc7O1qpVq/Tqq6/asbmAYUwIAADNE7RjQi688ELl5eXZtTkAANDG2RJCDh8+rKeffloJCQl2bA4AALQDfo8JOf5CdZZl6YcfflBUVJT+8pe/2FocAABou/wOIU899ZRPCAkLC9Ppp5+u5ORkdenSxdbiAABA2+V3CLnpppsCUAYAAGhv/B4Tsnz5cq1evbre8tWrV+vFF1+0pSgAAND2+R1CFi1apO7du9db3qNHDy1cuNCWogAAQNvndwjZs2ePevfuXW95r169tGfPHluKAgAAbZ/fIaRHjx76+OOP6y3fvn27unXrZktRAACg7fM7hEyePFl33nmn8vPz5Xa75Xa79fbbbysjI0PXXXddIGoEAABtkN9nxzzyyCP66quvdNlll6lDh9qnezweTZ06lTEhAACgyZp97ZjPP/9c27ZtU2RkpAYOHKhevXrZXVtAcO0YAACax+5jqN8tIXX69u2rvn37trgAAADQPvk9JuSaa67Ro48+Wm/5Y489pl/96le2FAUAANo+v0PIxo0bNW7cuHrLr7zySm3cuNGWogAAQNvndwiprKxUeHh4veUdO3ZURUWFLUUBAIC2z+8QMnDgQK1atare8pdfflnnnHOOLUUBAIC2z++BqfPmzVNaWpq++OILXXrppZKkvLw8ZWdnKycnx/YCAQBA2+R3CBk/frzWrl2rhQsXKicnR5GRkRo8eLDefvttde3aNRA1AgCANqjZ84TUqaio0MqVK7Vs2TJt2bJFbrfbrtoCgnlCAABoHruPoX6PCamzceNGTZs2TT179tQf/vAHXXrppfrggw9aXBAAAGgf/OqOKS0t1YoVK7Rs2TJVVFTo2muvVXV1tdauXcugVAAA4Jcmt4SMHz9e/fr108cff6ysrCx98803+uMf/xjI2gAAQBvW5JaQv//977rzzjt1xx13MF07AABosSa3hLz77rv64YcfNHToUCUnJ+uZZ57R/v37A1kbAABow5ocQi688EI9//zzKikp0W9+8xu9/PLL6tmzpzwej9avX68ffvghkHUCAIA2pkWn6O7atUvLli3TSy+9pIMHD+ryyy/Xa6+9Zmd9tuMUXQAAmidoTtGVpH79+umxxx7T3r17tXLlyhYXAwAA2o8WT1YWamgJAQCgeYKqJQQAAKC5CCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjAiKELJkyRIlJSWpU6dOSk5O1ubNmxtdd9SoUXI4HPVuv/jFL1qxYgAA0FLGQ8iqVas0e/ZszZ8/X1u3btXgwYOVmpqqffv2Nbh+bm6uSkpKvLdPPvlETqdTv/rVr1q5cgAA0BLGQ8iTTz6pW2+9VdOnT9c555yjpUuXKioqSi+88EKD63ft2lVxcXHe2/r16xUVFUUIAQAgxBgNITU1NdqyZYvGjBnjXRYWFqYxY8Zo06ZNTdrGsmXLdN1116lz584NPl5dXa2KigqfGwAAMM9oCNm/f7/cbrdiY2N9lsfGxqq0tPSkz9+8ebM++eQT3XLLLY2us2jRIsXExHhviYmJLa4bAAC0nPHumJZYtmyZBg4cqBEjRjS6zty5c1VeXu69FRUVtWKFAACgMR1Mvnj37t3ldDpVVlbms7ysrExxcXEnfG5VVZVefvllPfzwwydcLyIiQhERES2uFQAA2MtoS0h4eLiGDh2qvLw87zKPx6O8vDyNHDnyhM9dvXq1qqurdcMNNwS6TAAAEABGW0Ikafbs2Zo2bZqGDRumESNGKCsrS1VVVZo+fbokaerUqUpISNCiRYt8nrds2TJNnDhR3bp1M1E2AABoIeMhZNKkSfr222/14IMPqrS0VEOGDNG6deu8g1X37NmjsDDfBptdu3bp3Xff1T/+8Q8TJQMAABs4LMuyTBfRmioqKhQTE6Py8nJFR0ebLgcAgJBh9zE0pM+OAQAAoYsQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMKKD6QJCndvtVkFBgUpKShQfH6+UlBQ5nU7TZQEAEPQIIS2Qm5urjIwM7d2717vM5XJp8eLFSktLM1gZAADBj+6YZsrNzVV6erpPAJGk4uJipaenKzc311BlAACEBkJIM7jdbmVkZMiyrHqP1S3LzMyU2+1u7dIAAAgZhJBmKCgoqNcCcizLslRUVKSCgoJWrAoAgNBCCGmGkpISW9cDAKA9IoQ0Q3x8vK3rAQDQHhFCmiElJUUul0sOh6PBxx0OhxITE5WSktLKlQEAEDoIIc3gdDq1ePFiSaoXROruZ2VlMV8IAAAnQAhpprS0NOXk5CghIcFnucvlUk5ODvOEAABwEg6rofNM27CKigrFxMSovLxc0dHRLd4eM6YCANoLu4+hzJjaQk6nU6NGjTJdBgAAIYfuGAAAYAQhBAAAGEEIAQAARhBCAACAEYQQAABgBCEEAAAYQQgBAABGEEIAAIARhBAAAGAEIQQAABhBCAEAAEYQQgAAgBGEEAAAYAQhBAAAGEEIAQAARhBCAACAER1MF4A2xO2WCgqkkhIpPl5KSZGcTtNVAQCCFCEE9sjNlTIypL17/7PM5ZIWL5bS0szVBQAIWnTHoOVyc6X0dN8AIknFxbXLc3PN1AUACGqEELSM213bAmJZ9R+rW5aZWbseAADHIISgZQoK6reAHMuypKKi2vUAADgGIQQtU1Ji73oAgHaDEIKWiY+3dz0AQLtBCEHLpKTUngXjcDT8uMMhJSbWrgcAwDEIIWgZp7P2NFypfhCpu5+VxXwhAIB6CCFoubQ0KSdHSkjwXe5y1S5nnhAAQAOYrAz2SEuTJkxgxlQAQJMRQmAfp1MaNcp0FQCAEEF3DAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwwngIWbJkiZKSktSpUyclJydr8+bNJ1z/4MGDmjFjhuLj4xUREaGzzz5bb7zxRitVCwAA7NLB5IuvWrVKs2fP1tKlS5WcnKysrCylpqZq165d6tGjR731a2pqdPnll6tHjx7KyclRQkKCvv76a5122mmtXzwAAGgRh2VZlqkXT05O1vDhw/XMM89IkjwejxITEzVr1izde++99dZfunSpHn/8cX322Wfq2LFjk16jurpa1dXV3vsVFRVKTExUeXm5oqOj7XkjAAC0AxUVFYqJibHtGGqsO6ampkZbtmzRmDFj/lNMWJjGjBmjTZs2Nfic1157TSNHjtSMGTMUGxur8847TwsXLpTb7W70dRYtWqSYmBjvLTEx0fb3AgAA/GcshOzfv19ut1uxsbE+y2NjY1VaWtrgc7788kvl5OTI7XbrjTfe0Lx58/SHP/xBv/vd7xp9nblz56q8vNx7KyoqsvV9AACA5jE6JsRfHo9HPXr00HPPPSen06mhQ4equLhYjz/+uObPn9/gcyIiIhQREdHKlQIAgJMxFkK6d+8up9OpsrIyn+VlZWWKi4tr8Dnx8fHq2LGjnE6nd9mAAQNUWlqqmpoahYeHB7RmAABgH2PdMeHh4Ro6dKjy8vK8yzwej/Ly8jRy5MgGn3PxxRdr9+7d8ng83mX//ve/FR8fTwABACDEGJ0nZPbs2Xr++ef14osvaufOnbrjjjtUVVWl6dOnS5KmTp2quXPnete/4447dODAAWVkZOjf//63Xn/9dS1cuFAzZsww9RYAAEAzGR0TMmnSJH377bd68MEHVVpaqiFDhmjdunXewap79uxRWNh/clJiYqLefPNN3XXXXRo0aJASEhKUkZGhOXPmmHoLAACgmYzOE2KC3ec4AwDQXrSZeUIAAED7RggBAABGEEIAAIARhBAAAGAEIQQAABhBCAEAAEYQQgAAgBGEEAAAYAQhBAAAGEEIAQAARhi9dkyb4HZLBQVSSYkUHy+lpEhOp+mqAAAIeoSQlsjNlTIypL17/7PM5ZIWL5bS0szVBQBACKA7prlyc6X0dN8AIknFxbXLc3PN1AUAQIgghDSH213bAtLQBYjrlmVm1q4HAAAaRAhpjoKC+i0gx7Isqaiodj0AANAgQkhzlJTYux4AAO0QIaQ54uPtXQ8AgHaIENIcKSm1Z8E4HA0/7nBIiYm16wEAgAYRQprD6aw9DVeqH0Tq7mdlMV8IAAAnQAhprrQ0KSdHSkjwXe5y1S5nnhAAAE6IycpaIi1NmjCBGVMBAGgGQkhLOZ3SqFGmqwAAIOTQHQMAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADCCEAIAAIwghAAAACMIIQAAwAhCCAAAMIIQAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMCIoAghS5YsUVJSkjp16qTk5GRt3ry50XVXrFghh8Phc+vUqVMrVgsAAOxgPISsWrVKs2fP1vz587V161YNHjxYqamp2rdvX6PPiY6OVklJiff29ddft2LFAADADsZDyJNPPqlbb71V06dP1znnnKOlS5cqKipKL7zwQqPPcTgciouL895iY2NbsWIAAGCHDiZfvKamRlu2bNHcuXO9y8LCwjRmzBht2rSp0edVVlaqV69e8ng8uuCCC7Rw4UKde+65Da5bXV2t6upq7/3y8nJJUkVFhU3vAgCA9qHu2GlZli3bMxpC9u/fL7fbXa8lIzY2Vp999lmDz+nXr59eeOEFDRo0SOXl5XriiSd00UUX6V//+pdcLle99RctWqQFCxbUW56YmGjPmwAAoJ357rvvFBMT0+LtGA0hzTFy5EiNHDnSe/+iiy7SgAED9N///d965JFH6q0/d+5czZ4923vf4/HowIED6tatmxwOhy01VVRUKDExUUVFRYqOjrZlm2gdfHahi88udPHZha7y8nKdccYZ6tq1qy3bMxpCunfvLqfTqbKyMp/lZWVliouLa9I2OnbsqPPPP1+7d+9u8PGIiAhFRET4LDvttNOaVe/JREdHs0OFKD670MVnF7r47EJXWJg9Q0qNDkwNDw/X0KFDlZeX513m8XiUl5fn09pxIm63Wzt27FB8fHygygQAAAFgvDtm9uzZmjZtmoYNG6YRI0YoKytLVVVVmj59uiRp6tSpSkhI0KJFiyRJDz/8sC688EKdddZZOnjwoB5//HF9/fXXuuWWW0y+DQAA4CfjIWTSpEn69ttv9eCDD6q0tFRDhgzRunXrvINV9+zZ49Ps8/333+vWW29VaWmpunTpoqFDh+r999/XOeecY+otKCIiQvPnz6/X7YPgx2cXuvjsQhefXeiy+7NzWHadZwMAAOAH45OVAQCA9okQAgAAjCCEAAAAIwghAADACEKIDZYsWaKkpCR16tRJycnJ2rx5s+mScBIPPfSQHA6Hz61///6my0IDNm7cqPHjx6tnz55yOBxau3atz+OWZenBBx9UfHy8IiMjNWbMGH3++edmioWPk312N910U739cOzYsWaKhY9FixZp+PDhOvXUU9WjRw9NnDhRu3bt8lnnxx9/1IwZM9StWzedcsopuuaaa+pNPnoyhJAWWrVqlWbPnq358+dr69atGjx4sFJTU7Vv3z7TpeEkzj33XJWUlHhv7777rumS0ICqqioNHjxYS5YsafDxxx57TE8//bSWLl2qDz/8UJ07d1Zqaqp+/PHHVq4UxzvZZydJY8eO9dkPV65c2YoVojHvvPOOZsyYoQ8++EDr16/XkSNHdMUVV6iqqsq7zl133aX/+Z//0erVq/XOO+/om2++UVpamn8vZKFFRowYYc2YMcN73+12Wz179rQWLVpksCqczPz5863BgwebLgN+kmStWbPGe9/j8VhxcXHW448/7l128OBBKyIiwlq5cqWBCtGY4z87y7KsadOmWRMmTDBSD/yzb98+S5L1zjvvWJZVu5917NjRWr16tXednTt3WpKsTZs2NXm7tIS0QE1NjbZs2aIxY8Z4l4WFhWnMmDHatGmTwcrQFJ9//rl69uypPn366Prrr9eePXtMlwQ/FRYWqrS01GcfjImJUXJyMvtgiNiwYYN69Oihfv366Y477tB3331nuiQ0oLy8XJK8F67bsmWLjhw54rPv9e/fX2eccYZf+x4hpAX2798vt9vtnd21TmxsrEpLSw1VhaZITk7WihUrtG7dOj377LMqLCxUSkqKfvjhB9OlwQ91+xn7YGgaO3as/vznPysvL0+PPvqo3nnnHV155ZVyu92mS8MxPB6PMjMzdfHFF+u8886TVLvvhYeH17sgrL/7nvFp2wETrrzySu+/Bw0apOTkZPXq1Ut/+9vfdPPNNxusDGg/rrvuOu+/Bw4cqEGDBunMM8/Uhg0bdNlllxmsDMeaMWOGPvnkk4CMm6MlpAW6d+8up9NZbzRwWVmZ4uLiDFWF5jjttNN09tlna/fu3aZLgR/q9jP2wbahT58+6t69O/thEJk5c6b+93//V/n5+XK5XN7lcXFxqqmp0cGDB33W93ffI4S0QHh4uIYOHaq8vDzvMo/Ho7y8PI0cOdJgZfBXZWWlvvjiC8XHx5suBX7o3bu34uLifPbBiooKffjhh+yDIWjv3r367rvv2A+DgGVZmjlzptasWaO3335bvXv39nl86NCh6tixo8++t2vXLu3Zs8evfY/umBaaPXu2pk2bpmHDhmnEiBHKyspSVVWVpk+fbro0nMDdd9+t8ePHq1evXvrmm280f/58OZ1OTZ482XRpOE5lZaXPN+PCwkJt27ZNXbt21RlnnKHMzEz97ne/U9++fdW7d2/NmzdPPXv21MSJE80VDUkn/uy6du2qBQsW6JprrlFcXJy++OIL3XPPPTrrrLOUmppqsGpItV0w2dnZevXVV3Xqqad6x3nExMQoMjJSMTExuvnmmzV79mx17dpV0dHRmjVrlkaOHKkLL7yw6S9k92k87dEf//hH64wzzrDCw8OtESNGWB988IHpknASkyZNsuLj463w8HArISHBmjRpkrV7927TZaEB+fn5lqR6t2nTplmWVXua7rx586zY2FgrIiLCuuyyy6xdu3aZLRqWZZ34szt06JB1xRVXWKeffrrVsWNHq1evXtatt95qlZaWmi4bltXg5ybJWr58uXedw4cPW7/97W+tLl26WFFRUdbVV19tlZSU+PU6jp9eDAAAoFUxJgQAABhBCAEAAEYQQgAAgBGEEAAAYAQhBAAAGEEIAQAARhBCAACAEYQQAABgBCEEQJvgcDi0du1a02UA8AMhBECL3XTTTXI4HPVuY8eONV0agCDGBewA2GLs2LFavny5z7KIiAhD1QAIBbSEALBFRESE4uLifG5dunSRVNtV8uyzz+rKK69UZGSk+vTpo5ycHJ/n79ixQ5deeqkiIyPVrVs33XbbbaqsrPRZ54UXXtC5556riIgIxcfHa+bMmT6P79+/X1dffbWioqLUt29fvfbaa4F90wBahBACoFXMmzdP11xzjbZv367rr79e1113nXbu3ClJqqqqUmpqqrp06aKPPvpIq1ev1ltvveUTMp599lnNmDFDt912m3bs2KHXXntNZ511ls9rLFiwQNdee60+/vhjjRs3Ttdff70OHDjQqu8TgB9svfYvgHZp2rRpltPptDp37uxz+/3vf29ZVu1lwW+//Xaf5yQnJ1t33HGHZVmW9dxzz1ldunSxKisrvY+//vrrVlhYmPfS7j179rTuv//+RmuQZD3wwAPe+5WVlZYk6+9//7tt7xOAvRgTAsAWo0eP1rPPPuuzrGvXrt5/jxw50uexkSNHatu2bZKknTt3avDgwercubP38Ysvvlgej0e7du2Sw+HQN998o8suu+yENQwaNMj7786dOys6Olr79u1r7lsCEGCEEAC26Ny5c73uEbtERkY2ab2OHTv63Hc4HPJ4PIEoCYANGBMCoFV88MEH9e4PGDBAkjRgwABt375dVVVV3sffe+89hYWFqV+/fjr11FOVlJSkvLy8Vq0ZQGDREgLAFtXV1SotLfVZ1qFDB3Xv3l2StHr1ag0bNkw/+9nP9Ne//lWbN2/WsmXLJEnXX3+95s+fr2nTpumhhx7St99+q1mzZunGG29UbGysJOmhhx7S7bffrh49eujKK6/UDz/8oPfee0+zZs1q3TcKwDaEEAC2WLduneLj432W9evXT5999pmk2jNXXn75Zf32t79VfHy8Vq5cqXPOOUeSFBUVpTfffFMZGRkaPny4oqKidM011+jJJ5/0bmvatGn68ccf9dRTT+nuu+9W9+7dlZ6e3npvEIDtHJZlWaaLANC2ORwOrVmzRhMnTjRdCoAgwpgQAABgBCEEAAAYwZgQAAFHry+AhtASAgAAjCCEAAAAIwghAADACEIIAAAwghACAACMIIQAAAAjCCEAAMAIQggAADDi/wdDMSL99L9GogAAAABJRU5ErkJggg==", "text/plain": [ "
" ] @@ -7137,14 +7319,16 @@ " ylabel='Accuracy')\n", "ax.set_xticks(np.linspace(0, 20, 5).astype(int))\n", "ax.set_ylabel('Accuracy')\n", - "ax.set_ylim([0.5, 1])" + "ax.set_ylim([0.5, 1])\n" ] }, { "cell_type": "code", "execution_count": 93, - "id": "7c6d1072", - "metadata": {}, + "id": "be5a979e", + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [], "source": [ "del(lstm_model,\n", @@ -7152,12 +7336,12 @@ " lstm_logger,\n", " imdb_seq_dm,\n", " imdb_seq_train,\n", - " imdb_seq_test)" + " imdb_seq_test)\n" ] }, { "cell_type": "markdown", - "id": "7d179385", + "id": "240618c9", "metadata": {}, "source": [ "### Time Series Prediction\n", @@ -7169,7 +7353,7 @@ { "cell_type": "code", "execution_count": 94, - "id": "8e2d1d5c", + "id": "cf219016", "metadata": {}, "outputs": [], "source": [ @@ -7179,12 +7363,12 @@ " with_mean=True,\n", " with_std=True).fit_transform(NYSE[cols]),\n", " columns=NYSE[cols].columns,\n", - " index=NYSE.index)" + " index=NYSE.index)\n" ] }, { "cell_type": "markdown", - "id": "6ad4a748", + "id": "2ebf00d3", "metadata": {}, "source": [ "Next we set up the lagged versions of the data, dropping\n", @@ -7194,7 +7378,7 @@ { "cell_type": "code", "execution_count": 95, - "id": "6f6d0e12", + "id": "c0d6bb22", "metadata": {}, "outputs": [], "source": [ @@ -7204,12 +7388,12 @@ " newcol[lag:] = X[col].values[:-lag]\n", " X.insert(len(X.columns), \"{0}_{1}\".format(col, lag), newcol)\n", "X.insert(len(X.columns), 'train', NYSE['train'])\n", - "X = X.dropna()" + "X = X.dropna()\n" ] }, { "cell_type": "markdown", - "id": "91a924d2", + "id": "53d15ddf", "metadata": {}, "source": [ "Finally, we extract the response, training indicator, and drop the current day’s `DJ_return` and\n", @@ -7219,8 +7403,10 @@ { "cell_type": "code", "execution_count": 96, - "id": "d3c961ec", - "metadata": {}, + "id": "9de511d5", + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [ { "data": { @@ -7240,12 +7426,12 @@ "source": [ "Y, train = X['log_volume'], X['train']\n", "X = X.drop(columns=['train'] + cols)\n", - "X.columns" + "X.columns\n" ] }, { "cell_type": "markdown", - "id": "1c5f8d2f", + "id": "9c61eaa2", "metadata": {}, "source": [ "We first fit a simple linear model and compute the $R^2$ on the test data using\n", @@ -7255,7 +7441,7 @@ { "cell_type": "code", "execution_count": 97, - "id": "cb89da88", + "id": "89666a56", "metadata": {}, "outputs": [ { @@ -7277,7 +7463,7 @@ }, { "cell_type": "markdown", - "id": "b92425ab", + "id": "c3997039", "metadata": {}, "source": [ "We refit this model, including the factor variable `day_of_week`.\n", @@ -7288,8 +7474,10 @@ { "cell_type": "code", "execution_count": 98, - "id": "55d921d7", - "metadata": {}, + "id": "23ebf124", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [], "source": [ "X_day = pd.merge(X, \n", @@ -7299,7 +7487,7 @@ }, { "cell_type": "markdown", - "id": "afab736a", + "id": "8df4e475", "metadata": {}, "source": [ " Note that we do not have\n", @@ -7310,8 +7498,10 @@ { "cell_type": "code", "execution_count": 99, - "id": "4e87eeb9", - "metadata": {}, + "id": "4190a4dc", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [ { "data": { @@ -7331,7 +7521,7 @@ }, { "cell_type": "markdown", - "id": "43309e3b", + "id": "d98cc235", "metadata": {}, "source": [ "This model achieves an $R^2$ of about 46%." @@ -7339,7 +7529,7 @@ }, { "cell_type": "markdown", - "id": "93c61d80", + "id": "131bfc1c", "metadata": {}, "source": [ "To fit the RNN, we must reshape the data, as it will expect 5 lagged\n", @@ -7354,14 +7544,16 @@ "observation to be earliest in time, so we must reverse the current order. Hence we loop over\n", "`range(5,0,-1)` below, which is\n", "an example of using a `slice()` to index\n", - "iterable objects. The general notation is `start:end:step`." + "iterable objects. The general notation is `start:end:step`. " ] }, { "cell_type": "code", "execution_count": 100, - "id": "3689b318", - "metadata": {}, + "id": "a2b41f92", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [ { "data": { @@ -7384,12 +7576,12 @@ " for col in cols:\n", " ordered_cols.append('{0}_{1}'.format(col, lag))\n", "X = X.reindex(columns=ordered_cols)\n", - "X.columns" + "X.columns\n" ] }, { "cell_type": "markdown", - "id": "9dbcf9f7", + "id": "7b301214", "metadata": {}, "source": [ "We now reshape the data." @@ -7398,8 +7590,10 @@ { "cell_type": "code", "execution_count": 101, - "id": "5633e521", - "metadata": {}, + "id": "dde73c9e", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [ { "data": { @@ -7419,7 +7613,7 @@ }, { "cell_type": "markdown", - "id": "bd3ae3ff", + "id": "4fa6cd7a", "metadata": {}, "source": [ "By specifying the first size as -1, `numpy.reshape()` deduces its size based on the remaining arguments.\n", @@ -7434,7 +7628,7 @@ { "cell_type": "code", "execution_count": 102, - "id": "979a6066", + "id": "6f9d6357", "metadata": {}, "outputs": [], "source": [ @@ -7455,7 +7649,7 @@ }, { "cell_type": "markdown", - "id": "14671ab2", + "id": "a5d720e7", "metadata": {}, "source": [ "We fit the model in a similar fashion to previous networks. We\n", @@ -7465,13 +7659,14 @@ "early stopping, since then the test performance would be biased.\n", "\n", "We form the training dataset similar to\n", - "our `Hitters` example." + "our `Hitters` example.\n", + " " ] }, { "cell_type": "code", "execution_count": 103, - "id": "3528f4e7", + "id": "a21ca47e", "metadata": {}, "outputs": [], "source": [ @@ -7480,12 +7675,12 @@ " X_rnn_t = torch.tensor(X_rnn[mask].astype(np.float32))\n", " Y_t = torch.tensor(Y[mask].astype(np.float32))\n", " datasets.append(TensorDataset(X_rnn_t, Y_t))\n", - "nyse_train, nyse_test = datasets" + "nyse_train, nyse_test = datasets\n" ] }, { "cell_type": "markdown", - "id": "1a731099", + "id": "0d627892", "metadata": {}, "source": [ "Following our usual pattern, we inspect the summary." @@ -7494,8 +7689,10 @@ { "cell_type": "code", "execution_count": 104, - "id": "795c283e", - "metadata": {}, + "id": "8fa26b87", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [ { "data": { @@ -7530,15 +7727,14 @@ " input_data=X_rnn_t,\n", " col_names=['input_size',\n", " 'output_size',\n", - " 'num_params'])" + " 'num_params'])\n" ] }, { "cell_type": "markdown", - "id": "650e0ec7", + "id": "065db586", "metadata": {}, "source": [ - "\\newpage\n", "We again put the two datasets into a data module, with a\n", "batch size of 64." ] @@ -7546,8 +7742,10 @@ { "cell_type": "code", "execution_count": 105, - "id": "31640121", - "metadata": {}, + "id": "9b871361", + "metadata": { + "lines_to_next_cell": 0 + }, "outputs": [], "source": [ "nyse_dm = SimpleDataModule(nyse_train,\n", @@ -7559,7 +7757,7 @@ }, { "cell_type": "markdown", - "id": "1de7ae89", + "id": "8e6d61d8", "metadata": {}, "source": [ "We run some data through our model to be sure the sizes match up correctly." @@ -7568,7 +7766,7 @@ { "cell_type": "code", "execution_count": 106, - "id": "af6795f5", + "id": "b63d1f85", "metadata": {}, "outputs": [ { @@ -7586,12 +7784,12 @@ " out = nyse_model(x)\n", " print(y.size(), out.size())\n", " if idx >= 2:\n", - " break" + " break\n" ] }, { "cell_type": "markdown", - "id": "eecb6d9c", + "id": "7fc69ade", "metadata": {}, "source": [ "We follow our previous example for setting up a trainer for a\n", @@ -7602,7 +7800,7 @@ { "cell_type": "code", "execution_count": 107, - "id": "0d04344c", + "id": "c5f32a4f", "metadata": {}, "outputs": [], "source": [ @@ -7610,23 +7808,25 @@ " lr=0.001)\n", "nyse_module = SimpleModule.regression(nyse_model,\n", " optimizer=nyse_optimizer,\n", - " metrics={'r2':R2Score()})" + " metrics={'r2':R2Score()})\n" ] }, { "cell_type": "markdown", - "id": "9a10444d", + "id": "ed00be6a", "metadata": {}, "source": [ "Fitting the model should by now be familiar.\n", - "The results on the test data are very similar to the linear AR model." + "The results on the test data are very similar to the linear AR model. " ] }, { "cell_type": "code", "execution_count": 108, - "id": "485d7bb1", - "metadata": {}, + "id": "6bd98eb9", + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [ { "name": "stderr", @@ -7665,7 +7865,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "22d1e28154764ce2bc2b703feed55f16", + "model_id": "d9e7a14d99fa4708bc143ac70d601ad2", "version_major": 2, "version_minor": 0 }, @@ -8337,7 +8537,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6d1316d3520c4d8a81637fc86c3e3a44", + "model_id": "8b51741d91394be18445b204a45bb0e7", "version_major": 2, "version_minor": 0 }, @@ -8351,7 +8551,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e09ade307b3c49f582f74bf0dd4d2708", + "model_id": "a59a0ff31b4c4057b3b3f6ffc45984fb", "version_major": 2, "version_minor": 0 }, @@ -8365,7 +8565,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "48db0b82df9b4117a205a87c30592862", + "model_id": "f64aa56abbe74c17a1b9f9921d92efb5", "version_major": 2, "version_minor": 0 }, @@ -8379,7 +8579,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "adbabd20ada3402a8add185543f2aa93", + "model_id": "07ab034b948f4e718b84293f8b4c758b", "version_major": 2, "version_minor": 0 }, @@ -8393,7 +8593,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d9862f02df8b406aba5622da14bca716", + "model_id": "b3e47b1ef99e4b9d9e8134aa1f7ea5f6", "version_major": 2, "version_minor": 0 }, @@ -8407,7 +8607,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9db3fb30b2a34d5c8e9e38be897ef016", + "model_id": "a13a0be45dcc4cd282d22fc68a52f3ce", "version_major": 2, "version_minor": 0 }, @@ -8421,7 +8621,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "486e9671073a45439376ea092170d97d", + "model_id": "5d4b807459a94cdb9a82c1ce2f9b5b62", "version_major": 2, "version_minor": 0 }, @@ -8435,7 +8635,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "8ee6ba16c38343b1adbe9e5ca17d9b71", + "model_id": "7767365cd54b468f99303a5a4e17a674", "version_major": 2, "version_minor": 0 }, @@ -8449,7 +8649,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c1c59ee520094e7f9f760fb327a86f41", + "model_id": "f71a24b9e4f1491482a38ebb9bf9358b", "version_major": 2, "version_minor": 0 }, @@ -8463,7 +8663,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "58e4540611454682a163d168079e95ca", + "model_id": "ccfbeb14baab40f683da393300e51e3e", "version_major": 2, "version_minor": 0 }, @@ -8477,7 +8677,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4cea513833bc4d10b02b223dbe81adae", + "model_id": "5fba706618f54aed8b3f22326a5a707b", "version_major": 2, "version_minor": 0 }, @@ -8491,7 +8691,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "8d460ede9abf49269767f03cf58d8edf", + "model_id": "11a7bd9f7c814018912ed8d9532a1ba6", "version_major": 2, "version_minor": 0 }, @@ -8505,7 +8705,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "bfe141d501db4c2a90b56f58e113a57f", + "model_id": "535aaf6aaa764a7d930e492edf8a72a1", "version_major": 2, "version_minor": 0 }, @@ -8519,7 +8719,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6d2765bb28c044ee8815f9b3e016dc53", + "model_id": "3919d695caa64be1a070238078c109a6", "version_major": 2, "version_minor": 0 }, @@ -8533,7 +8733,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "adf839390a6e4323aa791c0e6ccaf093", + "model_id": "96c26fefad70422caf28c6abc375bcd1", "version_major": 2, "version_minor": 0 }, @@ -8547,7 +8747,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "164e87502e154f8496e23bb36f695a5c", + "model_id": "8e837dc54b2d479bafa1296901a8b01b", "version_major": 2, "version_minor": 0 }, @@ -8561,7 +8761,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e693c49f1b66454095b6d4a033f67c8f", + "model_id": "72b513b7ef4c44619a5cae4208f3e327", "version_major": 2, "version_minor": 0 }, @@ -8575,7 +8775,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c354d67aa1214fba8deb7e92e0597229", + "model_id": "081f4d85cd4b4ce182dfac23bb0392ad", "version_major": 2, "version_minor": 0 }, @@ -8589,7 +8789,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "dc24fdcb96304a8e834a63e6e3decc69", + "model_id": "43544f846dfc45fbb428ff51d46d2009", "version_major": 2, "version_minor": 0 }, @@ -8603,7 +8803,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "65f753269a0746a48d3c64fcdc215366", + "model_id": "05945ce98a4a4cb1a35701cc53d44c45", "version_major": 2, "version_minor": 0 }, @@ -8617,7 +8817,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "bf95d8cf83054bafa25a319786a2974e", + "model_id": "34ed1fc56d944d0b9dff1be64d960c09", "version_major": 2, "version_minor": 0 }, @@ -8631,7 +8831,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9d0316679ded44d79967c0716162bef0", + "model_id": "ee14ea26bf3f4d64ac8b7aaeea6f96c5", "version_major": 2, "version_minor": 0 }, @@ -8645,7 +8845,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2299886bfcf6422f9cf2877114f45cde", + "model_id": "8f3249bfb55a49e9a26481c08876109d", "version_major": 2, "version_minor": 0 }, @@ -8659,7 +8859,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b0993d9faf0f4cf282d173e182687e30", + "model_id": "cefa621ff7ca4dce9f91b6a02d8fc57d", "version_major": 2, "version_minor": 0 }, @@ -8673,7 +8873,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2a21d7ff2c984037a965c770089a068e", + "model_id": "55cd23b8837c45e18080a26e8cfc5312", "version_major": 2, "version_minor": 0 }, @@ -8687,7 +8887,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e18dc9f943a34fbb9334fc8ce90321bd", + "model_id": "c14d363f447741a0ac7ee3c94b339c69", "version_major": 2, "version_minor": 0 }, @@ -8701,7 +8901,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5b2e3ecdaf5d42a48576834fc2ee6cb5", + "model_id": "93415a4cd059468d9f525960aeb4dcb1", "version_major": 2, "version_minor": 0 }, @@ -8715,7 +8915,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e5e5c120b38f4ed9a448320fd22da2cf", + "model_id": "79b534a0b8864892805b4e95787560c4", "version_major": 2, "version_minor": 0 }, @@ -8729,7 +8929,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "8c508dd0f9d84d498cce077216c39820", + "model_id": "eecaf5b8b8c240f99bb3c71d439e9177", "version_major": 2, "version_minor": 0 }, @@ -8743,7 +8943,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "cfaa842436fa4012a3b6ee320648ae73", + "model_id": "329da07ffe7e438a8b6193beb9a4829b", "version_major": 2, "version_minor": 0 }, @@ -8757,7 +8957,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "cdf2e95cc4574f9aaf404204043aa97a", + "model_id": "d8ef86626f1148cf9ef7ff6a4468187a", "version_major": 2, "version_minor": 0 }, @@ -8771,7 +8971,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4cb22be8610b4a38b96ae996ed34e73d", + "model_id": "80a87f93b77c469889629ed53aa53b6e", "version_major": 2, "version_minor": 0 }, @@ -8785,7 +8985,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "507312865fe946b39cb9af3e5125741a", + "model_id": "ef7c7332ae604a1f98e1fac2196c9a93", "version_major": 2, "version_minor": 0 }, @@ -8799,7 +8999,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f96fc4385c874d8ca79641325be046aa", + "model_id": "28114adb5c79431b8b39c9bafc9a2e07", "version_major": 2, "version_minor": 0 }, @@ -8813,7 +9013,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7e7882cadd164182bf1ef2859144ef9b", + "model_id": "a45ec30a7ee5444d985a199449d07f4a", "version_major": 2, "version_minor": 0 }, @@ -8827,7 +9027,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "fc8fda5017434ff88a6d2c80b4668ed5", + "model_id": "838d70b621b1414a8a301aa0f942472a", "version_major": 2, "version_minor": 0 }, @@ -8841,7 +9041,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c45bbd31e5b7427a96c8dd8c98af4106", + "model_id": "9dfd7093769b48c6a6349abc41b63790", "version_major": 2, "version_minor": 0 }, @@ -8855,7 +9055,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "423d6e4ce2d944d7a6dcfb98832e8245", + "model_id": "299916f0445b4fb783d1b856a2cc66e5", "version_major": 2, "version_minor": 0 }, @@ -8869,7 +9069,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e63249d5fe55490bac8c7f044a954508", + "model_id": "106517aa5d284262bbc876cfe8e70bae", "version_major": 2, "version_minor": 0 }, @@ -8883,7 +9083,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3bb3ef58912845d6a2b344d0d27c76f7", + "model_id": "e85a54fbcd884230b286c390d0567066", "version_major": 2, "version_minor": 0 }, @@ -8897,7 +9097,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ee03bc001264473190844cca7c6d7339", + "model_id": "a92c88277c1b4fd0984674298d8e4114", "version_major": 2, "version_minor": 0 }, @@ -8911,7 +9111,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "17d01bb258774e83b1c6c60e79939c7c", + "model_id": "92f0d894359b420d98b27043f091ec35", "version_major": 2, "version_minor": 0 }, @@ -8925,7 +9125,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "93f9bac916484b3dbd9502955854b4c0", + "model_id": "a338797738da46b7b6d0cd25d02ed9b2", "version_major": 2, "version_minor": 0 }, @@ -8939,7 +9139,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b48db74fba7d4b37b0e38f34f23f2e70", + "model_id": "9e298e066c5440159e953a83dac6b0f7", "version_major": 2, "version_minor": 0 }, @@ -8953,7 +9153,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "99e6621b0cfd4db3a29f70f3642d1b59", + "model_id": "d062643a13584c3f938dae2dc4cf5a6c", "version_major": 2, "version_minor": 0 }, @@ -8967,7 +9167,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "52c52f17f81440d19db10f5eff12cb91", + "model_id": "e56648c37aaa4830b681120812fc5571", "version_major": 2, "version_minor": 0 }, @@ -8981,7 +9181,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "02c70a50998b4bf3a6788c11be92417e", + "model_id": "fecda147ed494bf4ab97d2ab1d354b3c", "version_major": 2, "version_minor": 0 }, @@ -8995,7 +9195,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6711aa16723f4fc3ad34c6b1502e09fd", + "model_id": "4091ffa653564f5f8230749202977d87", "version_major": 2, "version_minor": 0 }, @@ -9009,7 +9209,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "29b5fad5bc78460dbb7a975134b8df48", + "model_id": "fb0f5a32f0374aaca36437e175bc0b56", "version_major": 2, "version_minor": 0 }, @@ -9023,7 +9223,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3730fff3087c4a92acc043fd34dcfd5a", + "model_id": "27d8d4c63d5441d4aac53823318f1fd5", "version_major": 2, "version_minor": 0 }, @@ -9037,7 +9237,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "13aff3400dd34fe596d880f20e1e45c1", + "model_id": "60c8b51326cd4973974434d666428c16", "version_major": 2, "version_minor": 0 }, @@ -9051,7 +9251,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "92a1b2109d84436b8d2b31405e249c2b", + "model_id": "1311bf7148fe4c759e56a2f24098c853", "version_major": 2, "version_minor": 0 }, @@ -9065,7 +9265,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "43b9d30313804b479b63bb72db8c9d79", + "model_id": "4d532e6673584204b5112cf6829ccd07", "version_major": 2, "version_minor": 0 }, @@ -9079,7 +9279,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a42e12a6b8884a5ab178fbc9bfe174af", + "model_id": "c095eb3b627b4a88b05777529fdaa1fd", "version_major": 2, "version_minor": 0 }, @@ -9093,7 +9293,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a4c5db5ff62840178775a13adc5a4d86", + "model_id": "aac6482fc6964aa3a5135e4d548be1d9", "version_major": 2, "version_minor": 0 }, @@ -9107,7 +9307,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "01598aa40a714779ab9df04eea2807c7", + "model_id": "0dbd739c071a4163a8c5bc629c0b1ac3", "version_major": 2, "version_minor": 0 }, @@ -9121,7 +9321,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e54dd1f17efe4a4e84d2fcb87f0b6e0a", + "model_id": "a37ee23ad3d54b6eb4ac875dab871db2", "version_major": 2, "version_minor": 0 }, @@ -9135,7 +9335,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a90c01fd88f2481ca176afee6d958cd6", + "model_id": "05f2f9ae7b824f67a7cc71f28741e9ed", "version_major": 2, "version_minor": 0 }, @@ -9149,7 +9349,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "81e73db680de484f8b5579e76f5da475", + "model_id": "ff71147304644998aa31c250dcb43046", "version_major": 2, "version_minor": 0 }, @@ -9163,7 +9363,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f67a5b44f19b43b68c297ae3bf7035a7", + "model_id": "e691a634602c403c96c579efb6637671", "version_major": 2, "version_minor": 0 }, @@ -9177,7 +9377,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d42361a360e546b5a915b030533b570a", + "model_id": "8608effcdd7341308e5bd67fa43b4920", "version_major": 2, "version_minor": 0 }, @@ -9191,7 +9391,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "64ec77cf74db44a4a6b571722a05a5b7", + "model_id": "e6164d7b6d444f8e8045d52ed3997291", "version_major": 2, "version_minor": 0 }, @@ -9205,7 +9405,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c3c24a22a3cc42d799f95ecb206c43ab", + "model_id": "aff06b872bf54b4d8bea99e3d0308b14", "version_major": 2, "version_minor": 0 }, @@ -9219,7 +9419,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f4d14f180cff43a98ae9884d816f191a", + "model_id": "d01b320492d94fc184f39bf911d586ce", "version_major": 2, "version_minor": 0 }, @@ -9233,7 +9433,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "afea602c1398459680b8e403221a7022", + "model_id": "bd12cc563a5a40bd9b61d498e6e81f1f", "version_major": 2, "version_minor": 0 }, @@ -9247,7 +9447,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ffdc1edd916246e38f808382ec94fa77", + "model_id": "b14b068ce7284c42a7cb17e6e3c13d19", "version_major": 2, "version_minor": 0 }, @@ -9261,7 +9461,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f3589c31b2f4412c980d165a0f449b4d", + "model_id": "2f8799b83a464181be9e05b9f2c21022", "version_major": 2, "version_minor": 0 }, @@ -9275,7 +9475,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b45b6081dfe846c7a3a25cba5cbdc89d", + "model_id": "bf2b507fb0e44c2980ca169791350839", "version_major": 2, "version_minor": 0 }, @@ -9289,7 +9489,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f9ffc84bcee64d80affe0ea923826eeb", + "model_id": "135fa8c0920c460f9c5004c18719e333", "version_major": 2, "version_minor": 0 }, @@ -9303,7 +9503,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c8582f794ca74cef93c959e19d7480b1", + "model_id": "042d5ec03ed04e1bae17cc1d85985ea6", "version_major": 2, "version_minor": 0 }, @@ -9317,7 +9517,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f3c489c600254344a3ffd811faa0ac74", + "model_id": "2437fa7941214afe8fe18afe097cac21", "version_major": 2, "version_minor": 0 }, @@ -9331,7 +9531,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "86f59a171b7942b5af975d3e8a44c6ff", + "model_id": "34707aa6e61e4a1e8cbcf313b5ec21b7", "version_major": 2, "version_minor": 0 }, @@ -9345,7 +9545,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "1f75f82b43c2425f9031b44c4b612384", + "model_id": "250b2eec0e784dccb5b723e3bbab36fa", "version_major": 2, "version_minor": 0 }, @@ -9359,7 +9559,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "809ba7ea1dc041fb819682cc28f18e9f", + "model_id": "bc7518e85d404dea850bf4695ab7baad", "version_major": 2, "version_minor": 0 }, @@ -9373,7 +9573,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "551cf48fe8684342a1e184035a8facd7", + "model_id": "d531525bd69741af9f56ece0c04f7ba4", "version_major": 2, "version_minor": 0 }, @@ -9387,7 +9587,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "bd4935099f4548ec88818741aee8b271", + "model_id": "a8ebefd9c619414db273165828887c70", "version_major": 2, "version_minor": 0 }, @@ -9401,7 +9601,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "51717034d96b4c1a95b62b68afc48bf5", + "model_id": "6ed475af51844f7f83dac4f4ea1e9c38", "version_major": 2, "version_minor": 0 }, @@ -9415,7 +9615,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7495c693ad634c96b75203c41bbbf1bc", + "model_id": "c5c04f4ef765490f9323938b7680a57b", "version_major": 2, "version_minor": 0 }, @@ -9429,7 +9629,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f6c9315a181147c4a338ab7b64cc50b9", + "model_id": "a8aaa7d2d1b4472cb4af2cd40bd49e7e", "version_major": 2, "version_minor": 0 }, @@ -9443,7 +9643,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4c157eaad4f847c2bd103a4fb0c0f27f", + "model_id": "7579778ab51f436cb05a4ac2b126baf8", "version_major": 2, "version_minor": 0 }, @@ -9457,7 +9657,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7b470c701e99448e9926ed6e13339b0f", + "model_id": "70db7b6f37ca42d5a25e026b985b3cfd", "version_major": 2, "version_minor": 0 }, @@ -9471,7 +9671,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "0287131bc8db4499877bb84f959b06ab", + "model_id": "aa174f1a6f714f78afbdb49a8afa6c95", "version_major": 2, "version_minor": 0 }, @@ -9485,7 +9685,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "65acdbb7adf345348b7ed93f70278aaa", + "model_id": "1bc7a0cc7d4d4a5ebacf9b72bc9af24e", "version_major": 2, "version_minor": 0 }, @@ -9499,7 +9699,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d5d58325ff6e41bb992a7f30137e7d7e", + "model_id": "7f8633d560284e0a9bd5fde200d4679a", "version_major": 2, "version_minor": 0 }, @@ -9513,7 +9713,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3f9c15d370904aa4924818b2cc48d2b1", + "model_id": "2bbddc6d4536485aa5a644771e938415", "version_major": 2, "version_minor": 0 }, @@ -9527,7 +9727,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "1f2349c21cfe41ccbc9a85f7c38fa82b", + "model_id": "bab249c4740e47d9a24074f4073da858", "version_major": 2, "version_minor": 0 }, @@ -9541,7 +9741,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b9d60449f4834068835ff1f4e214cb8b", + "model_id": "436c139a4e944586a33bbfdc7b23d9c8", "version_major": 2, "version_minor": 0 }, @@ -9555,7 +9755,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "684f5577da304adca0ed50428958c5b6", + "model_id": "d3138d221aca46fc93f6ebaa5e80942b", "version_major": 2, "version_minor": 0 }, @@ -9569,7 +9769,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9426a34592374958a6830db35b5be455", + "model_id": "e178cfb64fa745ebb953a9f5af45fea6", "version_major": 2, "version_minor": 0 }, @@ -9583,7 +9783,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "36331dafcb2e4321a3ba3176d32cd6d3", + "model_id": "3fa089174b9941a5873fcb1a6a883c27", "version_major": 2, "version_minor": 0 }, @@ -9597,7 +9797,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "00867d1fe4074a4a94060228a8d90adb", + "model_id": "aa298a2b098e47fa8743575f7f677b02", "version_major": 2, "version_minor": 0 }, @@ -9611,7 +9811,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "eb1275a3013f411693e02a674c1a97d7", + "model_id": "23b3db99178d47c5a195db459bf16ca3", "version_major": 2, "version_minor": 0 }, @@ -9625,7 +9825,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7ffda7ad0e6b4bef8c0e133a533f01e2", + "model_id": "f27ad94021bf4dec9458aae7e2e2df8a", "version_major": 2, "version_minor": 0 }, @@ -9639,7 +9839,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "8d2fe06dca464bbdacbd445157d11a69", + "model_id": "39ac4527dcc747b18c341d63396f8cbe", "version_major": 2, "version_minor": 0 }, @@ -9653,7 +9853,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3b9f818cfc894bc387960642169849c2", + "model_id": "0b8b9ff48ac44b4bac75f542a60260f8", "version_major": 2, "version_minor": 0 }, @@ -9667,7 +9867,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "0944b209c0bb490aaea224741e168b00", + "model_id": "013b46faf9164907b4e472d8ca3aca76", "version_major": 2, "version_minor": 0 }, @@ -9681,7 +9881,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "34071d4607394ede94e70dccd6f191ea", + "model_id": "95559c72c85c4bb088268eeef4442ad3", "version_major": 2, "version_minor": 0 }, @@ -9695,7 +9895,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "711718e57a11470bafa601035a770667", + "model_id": "d380a7eb62ed4223bfc31fbebfb41f70", "version_major": 2, "version_minor": 0 }, @@ -9709,7 +9909,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "656688569890443c9b525e07aed5bbdd", + "model_id": "06f0c6b84f344b7c891643982bc16aad", "version_major": 2, "version_minor": 0 }, @@ -9723,7 +9923,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6bd13f71794b4c92b16685cfb65462e7", + "model_id": "1843c7e989d046dab2874d4b0d96fb7b", "version_major": 2, "version_minor": 0 }, @@ -9737,7 +9937,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "1845934ebf584fd898690c988062f631", + "model_id": "153b5df5136548b385eea2c5a5f7aa67", "version_major": 2, "version_minor": 0 }, @@ -9751,7 +9951,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3bea53084c6348c8a9e3810f6b6df93f", + "model_id": "62cb83fa75cd4458b205ef06ececa78f", "version_major": 2, "version_minor": 0 }, @@ -9765,7 +9965,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b8e07fe30b7c4de3953872cc6d8707b7", + "model_id": "856dbcac819e4ddfa6743562f20eb08b", "version_major": 2, "version_minor": 0 }, @@ -9779,7 +9979,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "955d8e66ae3b4d249dd5379b04128ab5", + "model_id": "dac1a9abbeb6486ca54ddcc802d937ec", "version_major": 2, "version_minor": 0 }, @@ -9793,7 +9993,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "467daf03861e4e7ea3922f147a396b31", + "model_id": "251a967ded744fb0a8e09b471d498547", "version_major": 2, "version_minor": 0 }, @@ -9807,7 +10007,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "270cd6462f704a12b040bfc166d58bf2", + "model_id": "70778cd9463d41e5ad3f3c62dfa1a76c", "version_major": 2, "version_minor": 0 }, @@ -9821,7 +10021,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b9eb27cfba7a42dfa1737a035fb2b1d9", + "model_id": "8ca3a2ee978b4616af599116e606b6ce", "version_major": 2, "version_minor": 0 }, @@ -9835,7 +10035,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "0d82a11a381c44a0ae50c66af8270f2b", + "model_id": "e628bedb201440efa7e8f2c3f8b6fb34", "version_major": 2, "version_minor": 0 }, @@ -9849,7 +10049,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9ab1f055cc9445c6887f6f63f6c771f3", + "model_id": "84f25391f10b48ebbc792f52f377880b", "version_major": 2, "version_minor": 0 }, @@ -9863,7 +10063,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5d91ec0662294c2ca2fe81263d08c856", + "model_id": "ccbf056a69e4468890f7eb78d7c4c04a", "version_major": 2, "version_minor": 0 }, @@ -9877,7 +10077,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "bfd7bad4ad454d5f9aad6243d408644a", + "model_id": "07b8ad4b6c8f4df69c98079cee686e42", "version_major": 2, "version_minor": 0 }, @@ -9891,7 +10091,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "fb94f59d34664990aa6b15c378901486", + "model_id": "c6ed79b2fe1043ba8bf5d1a404cdae6e", "version_major": 2, "version_minor": 0 }, @@ -9905,7 +10105,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c51363bea11e42f6b6a11110211cc7cf", + "model_id": "82f6919b65a342f49958f796718900f2", "version_major": 2, "version_minor": 0 }, @@ -9919,7 +10119,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "402719c56ad04e9a9d6f27a8c6e2b025", + "model_id": "4573bdb37f6d4e94a573642de7726182", "version_major": 2, "version_minor": 0 }, @@ -9933,7 +10133,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "453b43f568c74ab09132bbbbc47b2032", + "model_id": "a5ad6202dbcd417abf7af4989c10174d", "version_major": 2, "version_minor": 0 }, @@ -9947,7 +10147,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "fda864b6fcb64af6a6942e7f77a8bf63", + "model_id": "01840ff73cbe49b39fc79e34a07d155c", "version_major": 2, "version_minor": 0 }, @@ -9961,7 +10161,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b0385666500646b7be6c43fc87664b22", + "model_id": "de24af3869e248dca909a084e0f46c6d", "version_major": 2, "version_minor": 0 }, @@ -9975,7 +10175,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3810297772e94425a30584336d35ec62", + "model_id": "58bb85df795749df9c63121088b12dd9", "version_major": 2, "version_minor": 0 }, @@ -9989,7 +10189,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2e4190d3937b44fc9da3cf42e46ea8ec", + "model_id": "d18bf3c6f8e74eafada490271bea08b8", "version_major": 2, "version_minor": 0 }, @@ -10003,7 +10203,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "bac56efdc07c4b85936f1f4f3db354bf", + "model_id": "f9bb5e5285484e5ab65d8f6b94c9f136", "version_major": 2, "version_minor": 0 }, @@ -10017,7 +10217,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3098be5468fd4f3c9627dfffd465add6", + "model_id": "4cfbfaee681c416c90af5da1c2fb0669", "version_major": 2, "version_minor": 0 }, @@ -10031,7 +10231,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ddb20f6d47f14c6b909ab2ebb66221d9", + "model_id": "b4615d017bdb449b886b03851c87bad2", "version_major": 2, "version_minor": 0 }, @@ -10045,7 +10245,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "0a9dce491e91435aa857a54730dbd7f8", + "model_id": "6ba9dbe44af34462a5cd985830cfe685", "version_major": 2, "version_minor": 0 }, @@ -10059,7 +10259,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "74d05c1445f7456b83bc38ac1f3e9270", + "model_id": "b3441b0de6a54c3484a75faca8f58cb2", "version_major": 2, "version_minor": 0 }, @@ -10073,7 +10273,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "33d224d04eff456eb23008191c7f1f6e", + "model_id": "58dddf2c74fe47ddb01f45aa9abfd7c5", "version_major": 2, "version_minor": 0 }, @@ -10087,7 +10287,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "16e4e426509a4b538df82641595e050e", + "model_id": "90fd89faac2246b28c41264c1c9ce915", "version_major": 2, "version_minor": 0 }, @@ -10101,7 +10301,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "aa471b599d8f4a619d054c25185fbcfd", + "model_id": "bc560d03002c496cb574f1b674419e31", "version_major": 2, "version_minor": 0 }, @@ -10115,7 +10315,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "88a478ea905d4309af20eaecc87d17a0", + "model_id": "99a9c84cc3574e07af764af936e607d0", "version_major": 2, "version_minor": 0 }, @@ -10129,7 +10329,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "298a9e1561024b7cae72abbe31aa0ca7", + "model_id": "6e009baa1a7c4f2eb461c30f07435cf1", "version_major": 2, "version_minor": 0 }, @@ -10143,7 +10343,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d9c863491cff4c74a396ad0248ae618a", + "model_id": "8b2ada5f771d40e4aa46f6ae4dfcb1de", "version_major": 2, "version_minor": 0 }, @@ -10157,7 +10357,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2e6069ea1c5b40eb8fdf84b1b92550e6", + "model_id": "0c208f741d10494683a4d2459a9496f5", "version_major": 2, "version_minor": 0 }, @@ -10171,7 +10371,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d2563d6164494cf2b1556c3c29080334", + "model_id": "4aa46b30a1a5446eab8f506ac46a4806", "version_major": 2, "version_minor": 0 }, @@ -10185,7 +10385,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b3d56361504e496bba10e3dbae495f5c", + "model_id": "c5dbf784e2f54fca8ea0a0b1a11554c2", "version_major": 2, "version_minor": 0 }, @@ -10199,7 +10399,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4e639c35958d483ea17957d9f681ab1a", + "model_id": "1058670a04fc471b86aa4118dc13764e", "version_major": 2, "version_minor": 0 }, @@ -10213,7 +10413,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "491552be5a3c41dab502f0dbde9be3a6", + "model_id": "8551b6c74fe2478597fbe6256b38e66e", "version_major": 2, "version_minor": 0 }, @@ -10227,7 +10427,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5bfc341375c34341b5c79e3bfb9df2b2", + "model_id": "ad1d59ca4684479ebb83002e436d6ed6", "version_major": 2, "version_minor": 0 }, @@ -10241,7 +10441,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e99f29127ed84f1b856226554f14016f", + "model_id": "4e4fb50909b1446b81eac3ac190165d3", "version_major": 2, "version_minor": 0 }, @@ -10255,7 +10455,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "8874710edb7a49a9a09eee5202f5ed0a", + "model_id": "175bb491fc994a04b1068d65b8167c0d", "version_major": 2, "version_minor": 0 }, @@ -10269,7 +10469,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "56e0ea15c5a6455793b264c52c8a3b47", + "model_id": "f4fcd016265142efb1ecee6ff949d66a", "version_major": 2, "version_minor": 0 }, @@ -10283,7 +10483,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d838f30c979948afaa1ce77b9de75646", + "model_id": "515c9938b1da46e38e1d29fa334fe3a7", "version_major": 2, "version_minor": 0 }, @@ -10297,7 +10497,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d4da17c1d9fd4c868d155b04233a0823", + "model_id": "2778ca8a604b49799dab53fac0ffdcb6", "version_major": 2, "version_minor": 0 }, @@ -10311,7 +10511,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9ef663e69f7149dd8dca976f998ac074", + "model_id": "d4d85921ccbf4117893d7724382fa922", "version_major": 2, "version_minor": 0 }, @@ -10325,7 +10525,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7ddb0589665e412baa0be8e5051c00e7", + "model_id": "985d1c01680c430eb2c352d410d340b9", "version_major": 2, "version_minor": 0 }, @@ -10339,7 +10539,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b6f54290d17941ed982b62d647e042d3", + "model_id": "641bf5e4ebe140bf8e9c6b182ba1301a", "version_major": 2, "version_minor": 0 }, @@ -10353,7 +10553,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "674bc2bd23a94a7da258e0ffd55aa03d", + "model_id": "b6c1e3dd8e6a4d2f907672980fdf3fa2", "version_major": 2, "version_minor": 0 }, @@ -10367,7 +10567,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5314a2129238492f90e59c2edcee0cef", + "model_id": "01cd22445d104fe5a648091c9888769a", "version_major": 2, "version_minor": 0 }, @@ -10381,7 +10581,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "feec30a701694a7db8a3dc54eea6ef1e", + "model_id": "964324d105cd4dd8a5851f7ddc80f0aa", "version_major": 2, "version_minor": 0 }, @@ -10395,7 +10595,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "8a69c265c31f4ee699fd3394018a7487", + "model_id": "b3445e8c245f488bb43d0e4b26aa00c6", "version_major": 2, "version_minor": 0 }, @@ -10409,7 +10609,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9ec5cf4091604d76b434a488a8863f39", + "model_id": "4fa976eeb2104f7ca8bdee6f72642e6a", "version_major": 2, "version_minor": 0 }, @@ -10423,7 +10623,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3bc22174f1754a318316f42b9dddc4d8", + "model_id": "f23213f214774846beea60dd78775d5b", "version_major": 2, "version_minor": 0 }, @@ -10437,7 +10637,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f5d5780b2d914213a81e8f166358d9b3", + "model_id": "6669a571368746d6bc379b5bd765522e", "version_major": 2, "version_minor": 0 }, @@ -10451,7 +10651,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f90bc9e45531431395af4eec61d19057", + "model_id": "1eeb38cc37ea4947af33a622ea7963e8", "version_major": 2, "version_minor": 0 }, @@ -10465,7 +10665,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4e8dafa90d0347a9861aa13bcfb26338", + "model_id": "f1a9b899e49445acb4a25e79e640388d", "version_major": 2, "version_minor": 0 }, @@ -10486,7 +10686,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "25dbf0a142584c9ca70cb7376dd6c6f6", + "model_id": "1799f882afb24212b14aa28c3a80472c", "version_major": 2, "version_minor": 0 }, @@ -10498,16 +10698,29 @@ "output_type": "display_data" }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n", - "Runningstage.testing metric DataLoader 0\n", - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n", - " test_loss 0.6186699867248535\n", - " test_r2 0.4128519892692566\n", - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n" - ] + "data": { + "text/html": [ + "
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+       "┃   Runningstage.testing    ┃                           ┃\n",
+       "┃          metric           ┃       DataLoader 0        ┃\n",
+       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+       "│         test_loss         │    0.6186699867248535     │\n",
+       "│          test_r2          │    0.4128519892692566     │\n",
+       "└───────────────────────────┴───────────────────────────┘\n",
+       "
\n" + ], + "text/plain": [ + "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n", + "┃\u001b[1m \u001b[0m\u001b[1m Runningstage.testing \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m┃\n", + "┃\u001b[1m \u001b[0m\u001b[1m metric \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m DataLoader 0 \u001b[0m\u001b[1m \u001b[0m┃\n", + "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n", + "│\u001b[36m \u001b[0m\u001b[36m test_loss \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 0.6186699867248535 \u001b[0m\u001b[35m \u001b[0m│\n", + "│\u001b[36m \u001b[0m\u001b[36m test_r2 \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 0.4128519892692566 \u001b[0m\u001b[35m \u001b[0m│\n", + "└───────────────────────────┴───────────────────────────┘\n" + ] + }, + "metadata": {}, + "output_type": "display_data" }, { "data": { @@ -10532,7 +10745,7 @@ }, { "cell_type": "markdown", - "id": "dbd13ee3", + "id": "3c78b138", "metadata": {}, "source": [ "We could also fit a model without the `nn.RNN()` layer by just\n", @@ -10549,7 +10762,7 @@ { "cell_type": "code", "execution_count": 109, - "id": "91d6633a", + "id": "5586a609", "metadata": {}, "outputs": [], "source": [ @@ -10564,7 +10777,7 @@ }, { "cell_type": "markdown", - "id": "b5cf941c", + "id": "c1c361b4", "metadata": {}, "source": [ "Creating a data module follows a familiar pattern." @@ -10573,7 +10786,7 @@ { "cell_type": "code", "execution_count": 110, - "id": "87137503", + "id": "b72ecb13", "metadata": {}, "outputs": [], "source": [ @@ -10581,12 +10794,12 @@ " day_test,\n", " num_workers=min(4, max_num_workers),\n", " validation=day_test,\n", - " batch_size=64)" + " batch_size=64)\n" ] }, { "cell_type": "markdown", - "id": "2575f374", + "id": "32016e95", "metadata": {}, "source": [ "We build a `NonLinearARModel()` that takes as input the 20 features and a hidden layer with 32 units. The remaining steps are familiar." @@ -10595,7 +10808,7 @@ { "cell_type": "code", "execution_count": 111, - "id": "3cf09a55", + "id": "595df68b", "metadata": {}, "outputs": [], "source": [ @@ -10608,13 +10821,13 @@ " nn.Dropout(0.5),\n", " nn.Linear(32, 1))\n", " def forward(self, x):\n", - " return torch.flatten(self._forward(x))" + " return torch.flatten(self._forward(x))\n" ] }, { "cell_type": "code", "execution_count": 112, - "id": "930576f3", + "id": "e8142533", "metadata": {}, "outputs": [], "source": [ @@ -10623,12 +10836,12 @@ " lr=0.001)\n", "nl_module = SimpleModule.regression(nl_model,\n", " optimizer=nl_optimizer,\n", - " metrics={'r2':R2Score()})" + " metrics={'r2':R2Score()})\n" ] }, { "cell_type": "markdown", - "id": "a0e3043c", + "id": "86d4e139", "metadata": {}, "source": [ "We continue with the usual training steps, fit the model,\n", @@ -10638,9 +10851,9 @@ { "cell_type": "code", "execution_count": 113, - "id": "b9ae8d0e", + "id": "47bdb236", "metadata": { - "scrolled": true + "lines_to_next_cell": 0 }, "outputs": [ { @@ -10680,7 +10893,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c94ca7359dc64c5cae06933af0a60888", + "model_id": "adaf10fcbd8c4e5baddfc2ca65d32558", "version_major": 2, "version_minor": 0 }, @@ -10981,7 +11194,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "61bc63f45e9147da8547b39eb3654ff0", + "model_id": "2666f1d4f060468681384e1b122c9d30", "version_major": 2, "version_minor": 0 }, @@ -10993,16 +11206,29 @@ "output_type": "display_data" }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n", - "Runningstage.testing metric DataLoader 0\n", - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n", - " test_loss 0.5651810765266418\n", - " test_r2 0.4636155962944031\n", - "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n" - ] + "data": { + "text/html": [ + "
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+       "┃   Runningstage.testing    ┃                           ┃\n",
+       "┃          metric           ┃       DataLoader 0        ┃\n",
+       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+       "│         test_loss         │    0.5651810765266418     │\n",
+       "│          test_r2          │    0.4636155962944031     │\n",
+       "└───────────────────────────┴───────────────────────────┘\n",
+       "
\n" + ], + "text/plain": [ + "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n", + "┃\u001b[1m \u001b[0m\u001b[1m Runningstage.testing \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m┃\n", + "┃\u001b[1m \u001b[0m\u001b[1m metric \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m DataLoader 0 \u001b[0m\u001b[1m \u001b[0m┃\n", + "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n", + "│\u001b[36m \u001b[0m\u001b[36m test_loss \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 0.5651810765266418 \u001b[0m\u001b[35m \u001b[0m│\n", + "│\u001b[36m \u001b[0m\u001b[36m test_r2 \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 0.4636155962944031 \u001b[0m\u001b[35m \u001b[0m│\n", + "└───────────────────────────┴───────────────────────────┘\n" + ] + }, + "metadata": {}, + "output_type": "display_data" }, { "data": { @@ -11022,13 +11248,23 @@ "nl_trainer.fit(nl_module, datamodule=day_dm)\n", "nl_trainer.test(nl_module, datamodule=day_dm) " ] + }, + { + "cell_type": "markdown", + "id": "51f435d5", + "metadata": {}, + "source": [ + " \n", + "\n", + " \n" + ] } ], "metadata": { "jupytext": { "cell_metadata_filter": "-all", - "formats": "ipynb,md:myst", - "main_language": "python" + "main_language": "python", + "notebook_metadata_filter": "-all" }, "kernelspec": { "display_name": "Python3 (islp_freeze_311)", diff --git a/README.md b/README.md index 32e7cf1..8d77289 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # ISLP_labs +[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/intro-stat-learning/ISLP_labs/v2_devel) + + Up-to-date version of labs for ISLP. This repo will track labs for ISLP as their source code changes. The diff --git a/requirements.txt b/requirements.txt index 270dc42..952d8d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,4 @@ torchvision==0.15.2 pytorch-lightning==2.0.6 torchinfo==1.8.0 torchmetrics==1.0.1 - +ISLP=0.3.18