{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ritchie46/code/python/anaStruct/anastruct/fem/plotter.py:10: UserWarning: Plotly not found. The functions that require plotly will result in an error\n",
      "  warnings.warn(\"Plotly not found. The functions that require plotly will result in an error\")\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pickle\n",
    "import tensorflow as tf\n",
    "import sys\n",
    "sys.path.append(\"/home/ritchie46/code/python/anaStruct\")\n",
    "\n",
    "from anastruct.fem.system import SystemElements\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import math\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 20508 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fa71e957a90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ss = SystemElements()\n",
    "ss.add_element([2, 0])\n",
    "ss.add_element([1, 1])\n",
    "ss.add_element([0, 0])\n",
    "ss.add_element([[2, 0], [4, 0]])\n",
    "ss.add_element([[3, 1]])\n",
    "ss.add_element([[2, 0]])\n",
    "ss.add_element([[3, 1], [1, 1]])\n",
    "\n",
    "\n",
    "x_range = ss.nodes_range('x')\n",
    "length = max(x_range)\n",
    "middle_node_id = np.argmin(np.abs(np.array(x_range) - length // 2)) + 1\n",
    "\n",
    "ss.point_load(middle_node_id, Fz=-100)\n",
    "ss.show_structure()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         1.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  1.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  1.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  1.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.125     ,  0.375     ,  0.57142857,\n",
       "         0.85714286]), 0, False)"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "opposite = {\n",
    "    0:4,\n",
    "    1:5,\n",
    "    2:6,\n",
    "    3:7,\n",
    "    4:0,\n",
    "    5:1,\n",
    "    6:2,\n",
    "    7:3,\n",
    "}\n",
    "\n",
    "def scale_down(x, minimum, maximum):\n",
    "    \"\"\"\n",
    "    Scale the goal location down.\n",
    "    \n",
    "    :param x: (array) Location -> rows, collumns\n",
    "    :param minimum: (int) min(rows, col) -> 0\n",
    "    :param maximum: (int) max(rows, col)\n",
    "    \n",
    "    return: (array) normalized between 0 and 1\n",
    "    \"\"\"\n",
    "    return (x - minimum) / (maximum - minimum)\n",
    "\n",
    "def scale_up(x, minimum, maximum):\n",
    "    x * (maximum - minimum) + minimum\n",
    "\n",
    "class Environment:\n",
    "    def __init__(self, length=10, height=8, optimize='defl'):\n",
    "        self.length = length\n",
    "        self.height = height\n",
    "        self.optimize = optimize\n",
    "        self.state = None\n",
    "        self.location = None\n",
    "        self.deploy = None\n",
    "        self.state_count = None # Count the passed states by index (starting from 0)\n",
    "        # The state index at which the last element was added\n",
    "        self.last_element_added = None \n",
    "        self.last_element_location = None\n",
    "\n",
    "        # valid actions\n",
    "        self.no_action = None\n",
    "        self.action_space = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}\n",
    "        self.valid_actions = None\n",
    "        \n",
    "        # structure\n",
    "        self.ss = None\n",
    "        self.last_step = None\n",
    "        \n",
    "        # actions\n",
    "        self.right = 0\n",
    "        self.left = 4\n",
    "        self.up = 2\n",
    "        self.down = 6\n",
    "        self.up_right = 1\n",
    "        self.up_left = 3\n",
    "        self.down_right = 7\n",
    "        self.down_left = 5\n",
    "        \n",
    "        # If the state is a flattened array. This maps to the index displacements.\n",
    "        self.move_map = {self.right: np.array([0, 1]),\n",
    "                         self.left: np.array([0, -1]),\n",
    "                         self.up: np.array([-1, 0]),\n",
    "                         self.down: np.array([1, 0]),\n",
    "                         self.up_right: np.array([-1, 1]),\n",
    "                         self.up_left: np.array([-1, -1]),\n",
    "                         self.down_right: np.array([1, 1]),\n",
    "                         self.down_left: np.array([1, -1])} \n",
    "    \n",
    "    def reset(self):\n",
    "        self.ss = SystemElements()\n",
    "        self.last_step = 8\n",
    "        self.last_element_added = -1\n",
    "        self.last_element_location = np.array(self.location)\n",
    "        self.state_count = -1\n",
    "        \n",
    "        self.deploy = 1 # True\n",
    "        self.built_nodes = set()\n",
    "        \n",
    "        self.goal = np.array([self.height // 2, self.length - 1])\n",
    "        self.location = np.array([self.height // 2, 0])\n",
    "        \n",
    "        self.state = np.zeros((self.height, self.length))\n",
    "        self.state[np.split(self.location, 2)] = 1\n",
    "        self.det_valid_actions()\n",
    "        \n",
    "        return np.concatenate((self.state.ravel(), [self.deploy], scale_down(self.location, 0, max(self.height, self.length) - 1), \n",
    "                                   scale_down(self.goal, 0, max(self.height, self.length) -1)))\n",
    "    \n",
    "    def view(self):\n",
    "        s = np.array(self.state, dtype=object)\n",
    "        if np.all(self.location == self.goal):\n",
    "            s[np.split(self.goal, 2)] = \"R\"\n",
    "        else:\n",
    "            s[np.split(self.location, 2)] = \"L\"\n",
    "            s[np.split(self.goal, 2)] = \"G\"\n",
    "        return s\n",
    "    \n",
    "    def return_action(self, r):\n",
    "        done = False\n",
    "\n",
    "        # Bridge is build\n",
    "        if np.all(self.location == self.goal) and self.state[np.split(self.location, 2)]:\n",
    "            r = r + self.compute_reward()\n",
    "            done = True\n",
    "            return np.concatenate((self.state.ravel(), [self.deploy], scale_down(self.location, 0, max(self.height, self.length) -1), \n",
    "                                   scale_down(self.goal, 0, max(self.height, self.length) - 1))), r, done\n",
    "                    \n",
    "        self.det_valid_actions()\n",
    "        if len(self.valid_actions) == 0:\n",
    "            done = True\n",
    "            r -= 0.5\n",
    "        \n",
    "        return np.concatenate((self.state.ravel(), [self.deploy], scale_down(self.location, 0, max(self.height, self.length)), \n",
    "                                   scale_down(self.goal, 0, max(self.height - 1, self.length - 1)))), r, done\n",
    "    \n",
    "    def det_valid_actions(self):\n",
    "        no_action = set()\n",
    "        \n",
    "        # current location\n",
    "        row, col = self.location\n",
    "\n",
    "        # right:\n",
    "        if col + 3 >= self.length:\n",
    "            no_action.add(self.right)\n",
    "            no_action.add(self.up_right)\n",
    "            no_action.add(self.down_right)\n",
    "            \n",
    "        if col <= 2:\n",
    "            no_action.add(self.left)\n",
    "            no_action.add(self.up_left)\n",
    "            no_action.add(self.down_left)\n",
    "        \n",
    "        if row <= 2:\n",
    "            no_action.add(self.up)\n",
    "            no_action.add(self.up_right)\n",
    "            no_action.add(self.up_left)\n",
    " \n",
    "        if row >= self.height -3:\n",
    "            no_action.add(self.down)\n",
    "            no_action.add(self.down_right)\n",
    "            no_action.add(self.down_left)\n",
    "            \n",
    "        if self.deploy:\n",
    "            # make sure there are no double elements\n",
    "            for a, direction in self.move_map.items():\n",
    "                if a not in no_action:\n",
    "                    if int(self.state[np.split(self.location + direction, 2)]) == 1:\n",
    "                        no_action.add(a)\n",
    "        \n",
    "        if self.deploy:\n",
    "            no_action.add(8)\n",
    "        \n",
    "        self.no_action = no_action\n",
    "        self.valid_actions = list(self.action_space - no_action)\n",
    "        \n",
    "    \n",
    "    def step(self, a):\n",
    "        \"\"\"\n",
    "        :param a: (int) action direction\n",
    "        \n",
    "        → 0\n",
    "        ↗ 1\n",
    "        ↑ 2\n",
    "        ↖ 3\n",
    "        ← 4\n",
    "        ↙ 5\n",
    "        ↓ 6\n",
    "        ↘ 7\n",
    "        down: 8\n",
    "        \"\"\"\n",
    "        self.last_step = a\n",
    "        self.state_count += 1\n",
    "\n",
    "        if a in self.no_action:\n",
    "            return self.return_action(-0.1)\n",
    "            \n",
    "        if a == 8: # pre-deploy\n",
    "            if self.state[np.split(self.location, 2)] == 1:\n",
    "                self.deploy = 1\n",
    "            return self.return_action(+0.1)\n",
    "        \n",
    "        if self.deploy:            \n",
    "            y1 = self.height - 1 - self.location[0]\n",
    "            x1 = self.location[1]\n",
    "            self.state[np.split(self.location + self.move_map[a], 2)] = 1\n",
    "            self.state[np.split(self.location + self.move_map[a] * 2, 2)] = 1\n",
    "            self.state[np.split(self.location + self.move_map[a] * 3, 2)] = 1\n",
    "            \n",
    "            self.location += self.move_map[a] * 3\n",
    "                    \n",
    "            y2 = self.height - 1 - self.location[0]\n",
    "            x2 = self.location[1]\n",
    "            self.ss.add_element([[x1, y1], [x2, y2]])\n",
    "            self.deploy = 0 # bool\n",
    "            self.last_element_added = self.state_count\n",
    "            self.last_element_location = np.array(self.location)\n",
    "        else:\n",
    "            self.location += self.move_map[a] * 3 # update location\n",
    "\n",
    "        return self.return_action(0)\n",
    "    \n",
    "    def structure(self):\n",
    "        # last node's x\n",
    "        x_range = self.ss.nodes_range('x')\n",
    "        length = max(x_range)\n",
    "                \n",
    "        middle_node_id = np.argmin(np.abs(np.array(x_range) - length // 2)) + 1\n",
    "        max_node_id = np.argmin(np.abs(np.array(x_range) - length)) + 1\n",
    "        \n",
    "        self.ss.add_support_hinged(1)\n",
    "        self.ss.add_support_hinged(max_node_id)\n",
    "        \n",
    "        self.ss.point_load(middle_node_id, Fz=-100)\n",
    "        self.ss.solve()\n",
    "        \n",
    "        return abs(self.ss.get_node_displacements(middle_node_id)[\"uy\"])\n",
    "    \n",
    "    def compute_reward(self):\n",
    "        return 1 / (self.structure() / (100 * max(self.ss.nodes_range('x'))**3 / (48 * self.ss.EI)))\n",
    "    \n",
    "    def sample_action(self):\n",
    "        return np.random.randint(0, 9) \n",
    "    \n",
    "    def state_size(self):\n",
    "        return self.reset().size\n",
    "\n",
    "    \n",
    "\n",
    "        \n",
    "env = Environment(7, 8)\n",
    "env.reset()\n",
    "env.step(1)\n",
    "# env.step(8)\n",
    "# env.step(7)\n",
    "\n",
    "# # # env.step(8)\n",
    "# # print(env.view())\n",
    "# # print(env.location)\n",
    "\n",
    "# # int(env.state[np.split(env.location + np.array([ 1, 0]), 2)]) == 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://theneuralperspective.com/2016/11/25/reinforcement-learning-rl-policy-gradients-i/\n",
    "\n",
    "def leaky_relu(x, alpha=0.1):\n",
    "    return tf.maximum(x, alpha * x)\n",
    "\n",
    "class Agent:\n",
    "    def __init__(self, data_size, hidden_size, action_space, learning_rate, name):\n",
    "        \"\"\"\n",
    "        :param data_size: (int) Columns of the data vector.\n",
    "        :param hidden_size: (int) No. of hidden nodes.\n",
    "        :param action_space: (int) No. of outputs.\n",
    "        :param learning_rate: (flt)\n",
    "        \"\"\"\n",
    "        \n",
    "        if len(hidden_size) == 1:\n",
    "            one_hidden_layer = True\n",
    "            hidden_size.append(hidden_size[0])\n",
    "        else:\n",
    "            one_hidden_layer = False\n",
    "\n",
    "        \n",
    "        # Step 1: Feed forward\n",
    "        # The argmax is the maximum Q-value.\n",
    "        self.input_s = tf.placeholder(tf.float32, [None, data_size], name=\"input_s\")\n",
    "        self.w1 = tf.get_variable(f\"{name}_w1\", shape=[data_size, hidden_size[0]], initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.b1 = tf.get_variable(f\"{name}_b1\", shape=(hidden_size[0], ), initializer=tf.zeros_initializer())\n",
    "        self.layer_1 = leaky_relu(tf.matmul(self.input_s, self.w1) + self.b1)\n",
    "        \n",
    "        self.w2 = tf.get_variable(f\"{name}_w2\", shape=[hidden_size[0], hidden_size[1]], initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.b2 = tf.get_variable(f\"{name}_b2\", shape=(hidden_size[1], ), initializer=tf.zeros_initializer())\n",
    "        self.layer_2 = leaky_relu(tf.matmul(self.layer_1, self.w2) + self.b2)\n",
    "                     \n",
    "        self.w_out = tf.get_variable(f\"{name}_w_out\", shape=[hidden_size[1], action_space], initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.b_out = tf.get_variable(f\"{name}_b_out\", shape=(action_space, ), initializer=tf.zeros_initializer())\n",
    "        \n",
    "        if one_hidden_layer:\n",
    "            hidden_layer = self.layer_1\n",
    "        else:\n",
    "            hidden_layer = self.layer_2\n",
    "        \n",
    "        # argmax(Q(s, a)) \n",
    "        self.predict_Q = tf.matmul(hidden_layer, self.w_out) + self.b_out # actual Q-value\n",
    "        self.Q_a = tf.argmax(self.predict_Q, 1)\n",
    "        \n",
    "        # Boltzman exploration\n",
    "        self.temperature = tf.placeholder(dtype=tf.float32)\n",
    "        self.p = tf.nn.softmax(self.predict_Q / self.temperature)\n",
    "\n",
    "        \n",
    "        # Step 2: Determine loss / gradients. \n",
    "        # One hot encoded actions\n",
    "        self.executed_actions = tf.placeholder(tf.int32)\n",
    "        \n",
    "        self.one_hot = tf.one_hot(self.executed_actions, action_space)\n",
    "        self.Q = tf.reduce_sum(tf.multiply(self.predict_Q, self.one_hot), axis=1)\n",
    "        self.next_Q_r = tf.placeholder(tf.float32)\n",
    "\n",
    "         # Loss\n",
    "         # mse: (     target      -    prediction)^2\n",
    "         #      r + max(Q(s', a') -    Q(s, a) )^2\n",
    "        starter_learning_rate = learning_rate\n",
    "        self.train_count = tf.Variable(0, trainable=False, name=f\"{name}_train_count\")\n",
    "        learning_rate = tf.train.exponential_decay(starter_learning_rate, self.train_count,\n",
    "                                                   10000, 0.96)\n",
    "    \n",
    "        self.loss = tf.reduce_sum(tf.square(self.next_Q_r - self.Q))   \n",
    "        optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "        \n",
    "        self.train = optimizer.minimize(self.loss, self.train_count)\n",
    "        self.saver = tf.train.Saver()\n",
    "        \n",
    "        \n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "def discounted_reward(r, gamma):\n",
    "    \"\"\"\n",
    "    The reward for a given state. Is the reward for that state + the discounted sum of future rewards.\n",
    "    \n",
    "    :param r: (array) Rewards.\n",
    "    :param gamma: (flt) Discount factor\n",
    "    \"\"\"\n",
    "    return np.cumsum(r * gamma**(np.arange(len(r)))[::-1])[::-1]\n",
    "\n",
    "def prepare_update_target(trainables, tau=0.05):\n",
    "    \"\"\"\n",
    "    The weights and biases of the target will be a depended of the primary network.\n",
    "    \n",
    "    wb[target] = tau * wb[primary] + (1-tau) * wb[target]\n",
    "    \n",
    "    This is a tensorflow operation and still needs to be run with Session.run(operation_holder)\n",
    "    \n",
    "    :trainables (tf.trainable_variables)\n",
    "    :tau (flt) Rate to update the target graph\n",
    "    \"\"\"\n",
    "    \n",
    "    operation_holder = []\n",
    "    n_variables = len(trainables) // 2 # the agent has half and the target has half\n",
    "    \n",
    "    for i, v in enumerate(trainables[0: n_variables]):\n",
    "        operation_holder.append(trainables[i + n_variables].assign(v.value() * tau) +\n",
    "                                ((1 - tau) * trainables[i + n_variables].value()))\n",
    "    return operation_holder\n",
    "\n",
    "def update_target(operation_holder, session):\n",
    "    for op in operation_holder:\n",
    "        session.run(op)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[name: \"/cpu:0\"\n",
       " device_type: \"CPU\"\n",
       " memory_limit: 268435456\n",
       " locality {\n",
       " }\n",
       " incarnation: 1294037122547162787, name: \"/gpu:0\"\n",
       " device_type: \"GPU\"\n",
       " memory_limit: 235274240\n",
       " locality {\n",
       "   bus_id: 1\n",
       " }\n",
       " incarnation: 9790280558597191456\n",
       " physical_device_desc: \"device: 0, name: GeForce GTX 1050, pci bus id: 0000:01:00.0\"]"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.python.client import device_lib\n",
    "\n",
    "device_lib.list_local_devices()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.0368 train_count 35 loss 0.917629 temp 5 n_success 2\n",
      "-0.0464 train_count 71 loss 0.707885 temp 5 n_success 7\n",
      "-0.0363 train_count 93 loss 4.46655 temp 5 n_success 11\n",
      "-0.0305180651831 train_count 136 loss 4.16673 temp 5 n_success 16\n",
      "-0.0519 train_count 147 loss 0.522373 temp 5 n_success 17\n",
      "-0.0405995950298 train_count 156 loss 0.483245 temp 5 n_success 20\n",
      "-0.034 train_count 186 loss 19.7174 temp 5 n_success 24\n",
      "-0.0408 train_count 215 loss 0.693041 temp 5 n_success 27\n",
      "-0.0425 train_count 215 loss 0.693041 temp 5 n_success 27\n",
      "-0.0471 train_count 233 loss 2.18616 temp 5 n_success 33\n",
      "-0.0418 train_count 244 loss 0.41765 temp 5 n_success 36\n",
      "-0.0437 train_count 262 loss 0.402245 temp 5 n_success 42\n",
      "-0.041 train_count 287 loss 0.646051 temp 5 n_success 44\n",
      "-0.0446 train_count 337 loss 0.406382 temp 5 n_success 44\n",
      "-0.0325279997494 train_count 385 loss 1.30061 temp 5 n_success 46\n",
      "-0.0364 train_count 393 loss 0.391566 temp 5 n_success 50\n",
      "-0.0351829154513 train_count 430 loss 0.337913 temp 5 n_success 54\n",
      "-0.0288225409115 train_count 467 loss 0.325651 temp 5 n_success 57\n",
      "-0.039 train_count 501 loss 21.4267 temp 5 n_success 61\n",
      "-0.0315218877407 train_count 516 loss 1.52991 temp 5 n_success 65\n",
      "-0.0306439315509 train_count 520 loss 0.546605 temp 5 n_success 68\n",
      "-0.0456 train_count 540 loss 16.0435 temp 5 n_success 70\n",
      "-0.0388 train_count 562 loss 3.53799 temp 5 n_success 74\n",
      "-0.051 train_count 582 loss 22.6095 temp 5 n_success 77\n",
      "-0.0492 train_count 608 loss 0.276391 temp 5 n_success 79\n",
      "-0.0511 train_count 634 loss 0.424335 temp 5 n_success 82\n",
      "-0.0382225409115 train_count 655 loss 0.311872 temp 5 n_success 91\n",
      "-0.0289477250232 train_count 684 loss 0.40743 temp 4.8911473364437015 n_success 96\n",
      "-0.0541 train_count 723 loss 0.3027 temp 4.652486874766168 n_success 102\n",
      "-0.043631535754 train_count 762 loss 0.611782 temp 4.4254717003903945 n_success 106\n",
      "-0.0342 train_count 765 loss 0.372836 temp 4.209533588838029 n_success 112\n",
      "-0.0462 train_count 765 loss 0.372836 temp 4.004132041787183 n_success 114\n",
      "-0.0451 train_count 791 loss 0.244125 temp 3.808752934192038 n_success 116\n",
      "-0.0353363016068 train_count 839 loss 0.418495 temp 3.6229072274154217 n_success 119\n",
      "-0.0474 train_count 889 loss 0.396872 temp 3.4461297451532507 n_success 121\n",
      "-0.034 train_count 939 loss 0.386683 temp 3.277978009086972 n_success 122\n",
      "-0.0303715890197 train_count 973 loss 0.270768 temp 3.1180311313496274 n_success 127\n",
      "-0.0369 train_count 1002 loss 5.73855 temp 2.965888761033321 n_success 130\n",
      "-0.0443225409115 train_count 1041 loss 50.1138 temp 2.8211700821012147 n_success 134\n",
      "-0.0425 train_count 1064 loss 0.478546 temp 2.6835128601957567 n_success 139\n",
      "-0.0389 train_count 1064 loss 0.478546 temp 2.5525725359573173 n_success 141\n",
      "-0.0263487723197 train_count 1072 loss 0.284453 temp 2.4280213625837694 n_success 144\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-203-e0fd4d7a071e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    114\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m1750\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mcount\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    115\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 116\u001b[0;31m             \u001b[0mbatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvstack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    117\u001b[0m             \u001b[0mbatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m150\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    118\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/miniconda3/lib/python3.6/site-packages/numpy/core/shape_base.py\u001b[0m in \u001b[0;36mvstack\u001b[0;34m(tup)\u001b[0m\n\u001b[1;32m    235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    236\u001b[0m     \"\"\"\n\u001b[0;32m--> 237\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_nx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0matleast_2d\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_m\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0m_m\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtup\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    238\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    239\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mhstack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtup\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/miniconda3/lib/python3.6/site-packages/numpy/core/shape_base.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    236\u001b[0m     \"\"\"\n\u001b[0;32m--> 237\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_nx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0matleast_2d\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_m\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0m_m\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtup\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    238\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    239\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mhstack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtup\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/miniconda3/lib/python3.6/site-packages/numpy/core/shape_base.py\u001b[0m in \u001b[0;36matleast_2d\u001b[0;34m(*arys)\u001b[0m\n\u001b[1;32m    104\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mary\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mary\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 106\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mary\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnewaxis\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    107\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    108\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mary\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "count = 0\n",
    "\n",
    "env = Environment(13, 6)\n",
    "with tf.device('/gpu:0'):\n",
    "    from collections import deque    \n",
    "    env.reset()\n",
    "\n",
    "    H = [128, 64] # hidden neurons\n",
    "    D = env.state_size() # input (state of the environment)\n",
    "    learning_rate = 1e-3\n",
    "    gamma = 0.99 # discount factor\n",
    "    epochs = 5000\n",
    "    buffer_size = 300000\n",
    "    max_frames = 150\n",
    "    action_space = 9\n",
    "\n",
    "    contin = 1\n",
    "\n",
    "    if not contin:\n",
    "        eps = 0.3\n",
    "        temp = 5\n",
    "        tf.reset_default_graph()\n",
    "        agent = Agent(D, H, action_space, learning_rate, \"agent\")\n",
    "        target = Agent(D, H, action_space, learning_rate, \"target\")\n",
    "\n",
    "        init = tf.global_variables_initializer()\n",
    "        sess = tf.Session()\n",
    "        sess.run(init)\n",
    "        buffer = []\n",
    "        n_success = 0\n",
    "\n",
    "        # the first half of the list are the variables of the agent.\n",
    "        # The last half the variables of the target\n",
    "        variables = tf.trainable_variables() \n",
    "        operation_holder = prepare_update_target(variables)\n",
    "        scores = list()\n",
    "        success_buffer = deque()\n",
    "\n",
    "    last_ep = 0\n",
    "    train_count = 0\n",
    "    loss = 0\n",
    "\n",
    "    for ep in range(epochs):\n",
    "\n",
    "        if temp > 1 and len(buffer) >= buffer_size:\n",
    "            temp *= 0.999\n",
    "\n",
    "        if (ep + 1) % 50 == 0 and len(scores) > 1000:\n",
    "            print(np.mean(scores[-1000:]), \"train_count\", train_count,\n",
    "                  \"loss\", loss, \"temp\", temp, \"n_success\", n_success)\n",
    "\n",
    "            scores = scores[-1000:]\n",
    "\n",
    "        s = env.reset()\n",
    "        s = [s]\n",
    "        \n",
    "        episode_buffer = []\n",
    "\n",
    "        for c in range(max_frames):\n",
    "            count += 1\n",
    "\n",
    "            p = sess.run(agent.p, {agent.input_s: s, agent.temperature: temp})\n",
    "            a = np.random.choice(p[0], p=p[0])\n",
    "            \n",
    "            a = np.argmin(np.abs(p[0] - a))\n",
    "#             a = np.argmax(p[0] == a)\n",
    "\n",
    "            s_new, r, done = env.step(a)\n",
    "\n",
    "            buffer.append([s, a, r, s_new, done])\n",
    "            episode_buffer.append([s, a, s_new])\n",
    "\n",
    "            scores.append(r)\n",
    "            \n",
    "            s = [s_new]\n",
    "            \n",
    "            if done:\n",
    "                if r > 0:\n",
    "                    n_success += 1\n",
    "                break\n",
    "                \n",
    "        # Hindsight Eperience Replay\n",
    "        \n",
    "        if env.last_element_location is not None and max(env.ss.nodes_range('x')) > 3: # Though very rare. No valid action is taken\n",
    "            index = env.last_element_added\n",
    "            i = 0\n",
    "\n",
    "            goal = scale_down(env.last_element_location, 0, max(env.height, env.length) - 1)\n",
    "            while i < index:\n",
    "                i += 1\n",
    "                \n",
    "                # retreive the states from the episode buffer\n",
    "                s, a, s_new = episode_buffer[i]\n",
    "                s = s[0] # remove redundant brackets\n",
    "                \n",
    "                # replace the goal in the state with the 'fake' goal\n",
    "                np.put(s, [-2, -1], goal)\n",
    "                np.put(s_new, [-2, -1], goal)\n",
    "                 \n",
    "                if i == index:\n",
    "                    done = True\n",
    "                    r = env.compute_reward()                       \n",
    "                       \n",
    "                else:\n",
    "                    done = False\n",
    "                    r = 0\n",
    "                \n",
    "                buffer.append([s, a, r, s_new, done])\n",
    "\n",
    "            if len(buffer) > buffer_size:\n",
    "                buffer = buffer[-buffer_size:]\n",
    "            \n",
    "        # Train\n",
    "        if len(buffer) >= 1750 and count % 2 == 0:\n",
    "\n",
    "            batch = np.vstack(buffer)\n",
    "            batch = batch[np.random.randint(0, len(buffer), size=150)]\n",
    "\n",
    "            s_ = np.vstack(batch[:, 0])\n",
    "            s_new_ = np.vstack(batch[:, 3])\n",
    "            r_ = batch[:, 2]\n",
    "            a_ = batch[:, 1]\n",
    "            done_ = np.array(batch[:, 4], dtype=bool)\n",
    "            Q = sess.run(agent.predict_Q, {agent.input_s: s_})\n",
    "\n",
    "\n",
    "            # Double Q-learning primary network chooses an action:\n",
    "            Q_i = np.argmax(sess.run(agent.predict_Q, {agent.input_s: s_new_}), 1)\n",
    "\n",
    "            # Target network produces the Q-value of the chosen action\n",
    "            Q_new = sess.run(target.predict_Q, {target.input_s: s_new_})\n",
    "            max_Q_new =Q_new[np.arange(Q_i.size)[:, None], Q_i[:, None]][:, -1]\n",
    "\n",
    "            target_Q = r_ + gamma * max_Q_new\n",
    "            target_Q[done_] = r_[done_]\n",
    "\n",
    "            train_count, loss, _ = sess.run([agent.train_count, agent.loss, agent.train], \n",
    "                                            feed_dict={agent.input_s: s_, \n",
    "                                                       agent.executed_actions: a_, \n",
    "                                                       agent.next_Q_r: target_Q})\n",
    "            if count % 4 == 0:\n",
    "                # update target network\n",
    "                update_target(operation_holder, sess)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/ritchie46/Dropbox/Code/Python/deep_learning/vanilla/reinforcement_learning/models/model_her/model.ckpt'"
      ]
     },
     "execution_count": 270,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agent.saver.save(sess, \n",
    "                 \"/home/ritchie46/Dropbox/Code/Python/deep_learning/vanilla/reinforcement_learning/\" +\n",
    "                 \"models/model_her/model.ckpt\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " [[0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " ['L' 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 'G']\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]]\n",
      "[[ 0.03744602 -0.02888978 -0.01323947 -0.1172988  -0.07983553 -0.07818529\n",
      "  -0.05138375 -0.02168381 -0.07894801]] [0, 1, 2, 9]\n",
      "action 0\n",
      "0 False\n",
      "\n",
      " [[0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [1.0 1.0 1.0 'L' 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 'G']\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]]\n",
      "[[ 0.00611979 -0.01767148 -0.00338707 -0.0279732  -0.02104705 -0.10892555\n",
      "  -0.0905954  -0.06917382  0.06944042]] [0, 1, 2, 3, 4, 8, 9]\n",
      "action 8\n",
      "0.1 False\n",
      "\n",
      " [[0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [1.0 1.0 1.0 'L' 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 'G']\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]]\n",
      "[[ 0.02476527 -0.03091978 -0.034238   -0.08606761 -0.08230525 -0.08085909\n",
      "  -0.04030796  0.02056772 -0.08270087]] [0, 1, 2, 3, 9]\n",
      "action 0\n",
      "0 False\n",
      "\n",
      " [[0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [1.0 1.0 1.0 1.0 1.0 1.0 'L' 0.0 0.0 0.0 0.0 0.0 'G']\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]]\n",
      "[[ 0.03326266 -0.018729   -0.02351833 -0.01548243 -0.03740275 -0.09187511\n",
      "  -0.05730958 -0.02471527  0.09445465]] [0, 1, 2, 3, 4, 8, 9]\n",
      "action 8\n",
      "0.1 False\n",
      "\n",
      " [[0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [1.0 1.0 1.0 1.0 1.0 1.0 'L' 0.0 0.0 0.0 0.0 0.0 'G']\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]]\n",
      "[[ 0.13147281 -0.00361413  0.04649895 -0.0314498  -0.05390706 -0.05526683\n",
      "  -0.01445211  0.08924725 -0.04301133]] [0, 1, 2, 3, 9]\n",
      "action 0\n",
      "0 False\n",
      "\n",
      " [[0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 'L' 0.0 0.0 'G']\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]]\n",
      "[[ 0.10093129 -0.01788209  0.0408037   0.01912121  0.01089984 -0.02279634\n",
      "   0.01150069  0.10682753  0.03775209]] [0, 1, 2, 3, 4, 8, 9]\n",
      "Needed a 0th try\n",
      "action 0\n",
      "0 False\n",
      "\n",
      " [[0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 'R']\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]]\n",
      "[[ 0.09293883 -0.02704024  0.03529771  0.01310346  0.0139786  -0.01785705\n",
      "   0.02663107  0.11546389  0.03714491]] [2, 3, 4, 8, 9]\n",
      "Needed a 0th try\n",
      "Needed a 1th try\n",
      "action 8\n",
      "0.1 False\n",
      "\n",
      " [[0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 'R']\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]]\n",
      "[[ 0.09293883 -0.02704024  0.03529771  0.01310346  0.0139786  -0.01785705\n",
      "   0.02663107  0.11546389  0.03714491]] [2, 3, 4, 8, 9]\n",
      "Needed a 0th try\n",
      "Needed a 1th try\n",
      "action 8\n",
      "0.1 False\n",
      "\n",
      " [[0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 'R']\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]]\n",
      "[[ 0.09293883 -0.02704024  0.03529771  0.01310346  0.0139786  -0.01785705\n",
      "   0.02663107  0.11546389  0.03714491]] [2, 3, 4, 8, 9]\n",
      "Needed a 0th try\n",
      "Needed a 1th try\n",
      "action 8\n",
      "0.1 False\n",
      "\n",
      " [[0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 'R']\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]]\n",
      "[[ 0.09293883 -0.02704024  0.03529771  0.01310346  0.0139786  -0.01785705\n",
      "   0.02663107  0.11546389  0.03714491]] [2, 3, 4, 8, 9]\n",
      "Needed a 0th try\n",
      "Needed a 1th try\n",
      "action 8\n",
      "0.1 False\n",
      "\n",
      " [[0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 'R']\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]]\n",
      "[[ 0.09293883 -0.02704024  0.03529771  0.01310346  0.0139786  -0.01785705\n",
      "   0.02663107  0.11546389  0.03714491]] [2, 3, 4, 8, 9]\n",
      "Needed a 0th try\n",
      "Needed a 1th try\n",
      "action 8\n",
      "0.1 False\n",
      "\n",
      " [[0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 0.0 0.0 'R']\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]\n",
      " [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]]\n",
      "[[ 0.09293883 -0.02704024  0.03529771  0.01310346  0.0139786  -0.01785705\n",
      "   0.02663107  0.11546389  0.03714491]] [2, 3, 4, 8, 9]\n",
      "Needed a 0th try\n",
      "Needed a 1th try\n",
      "action 8\n",
      "0.1 False\n",
      "\r",
      " 0.7999999999999999 0.24\n"
     ]
    }
   ],
   "source": [
    "env = Environment(13, 6, \"moment\")\n",
    "s = env.reset()\n",
    "\n",
    "\n",
    "\"\"\"    \n",
    "    → 0\n",
    "    ↗ 1\n",
    "    ↑ 2\n",
    "    ↖ 3\n",
    "    ← 4\n",
    "    ↙ 5\n",
    "    ↓ 6\n",
    "    ↘ 7\n",
    "\"\"\"\n",
    "\n",
    "bot = agent\n",
    "\n",
    "total_r = 0\n",
    "\n",
    "for i in range(12):    \n",
    "    print(\"\\n\", env.view())\n",
    "    \n",
    "    a_dst = sess.run(bot.predict_Q, {bot.input_s: [s]})\n",
    "    print(a_dst, env.valid_actions)\n",
    "    \n",
    "    for j in range(8):\n",
    "        a = np.argmax(a_dst)\n",
    "        if a not in env.valid_actions:\n",
    "            print(f\"Needed a {j}th try\")\n",
    "            a_dst[0][a] = - 10\n",
    "        else:\n",
    "            break\n",
    "\n",
    "    \n",
    "    s, r, d = env.step(a)\n",
    "\n",
    "    print(\"action\", a)\n",
    "    print(r, d)\n",
    "    total_r += r\n",
    "    \n",
    "    if d == True:\n",
    "        env.ss.show_bending_moment()\n",
    "        break\n",
    "\n",
    "print(\"\\r\", total_r, env.structure())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " [[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 1.  1.  1.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]]\n",
      "[[-0.01786325 -0.055416   -0.08617482 -0.08815897 -0.03658652 -0.08668041\n",
      "  -0.0850154  -0.08539797 -0.04319948]]\n",
      "action 0\n",
      "0 False\n",
      "\n",
      " [[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 1.  1.  1.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]]\n",
      "[[-0.02396103 -0.05518523 -0.08627115 -0.07779825 -0.02273003 -0.07515369\n",
      "  -0.06405259 -0.08325803 -0.04428619]]\n",
      "action 4\n",
      "0 False\n",
      "\n",
      " [[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 1.  1.  1.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]]\n",
      "[[-0.01786325 -0.055416   -0.08617482 -0.08815897 -0.03658652 -0.08668041\n",
      "  -0.0850154  -0.08539797 -0.04319948]]\n",
      "action 0\n",
      "0 False\n",
      "\n",
      " [[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 1.  1.  1.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]]\n",
      "[[-0.02396103 -0.05518523 -0.08627115 -0.07779825 -0.02273003 -0.07515369\n",
      "  -0.06405259 -0.08325803 -0.04428619]]\n",
      "action 4\n",
      "0 False\n",
      "\n",
      " [[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 1.  1.  1.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]]\n",
      "[[-0.01786325 -0.055416   -0.08617482 -0.08815897 -0.03658652 -0.08668041\n",
      "  -0.0850154  -0.08539797 -0.04319948]]\n",
      "action 0\n",
      "0 False\n",
      "\n",
      " [[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 1.  1.  1.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]]\n",
      "[[-0.02396103 -0.05518523 -0.08627115 -0.07779825 -0.02273003 -0.07515369\n",
      "  -0.06405259 -0.08325803 -0.04428619]]\n",
      "action 4\n",
      "0 False\n",
      "\n",
      " [[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 1.  1.  1.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]]\n",
      "[[-0.01786325 -0.055416   -0.08617482 -0.08815897 -0.03658652 -0.08668041\n",
      "  -0.0850154  -0.08539797 -0.04319948]]\n",
      "action 0\n",
      "0 False\n",
      "\n",
      " [[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 1.  1.  1.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]]\n",
      "[[-0.02396103 -0.05518523 -0.08627115 -0.07779825 -0.02273003 -0.07515369\n",
      "  -0.06405259 -0.08325803 -0.04428619]]\n",
      "action 4\n",
      "0 False\n",
      "\n",
      " [[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 1.  1.  1.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]]\n",
      "[[-0.01786325 -0.055416   -0.08617482 -0.08815897 -0.03658652 -0.08668041\n",
      "  -0.0850154  -0.08539797 -0.04319948]]\n",
      "action 0\n",
      "0 False\n",
      "\n",
      " [[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 1.  1.  1.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]]\n",
      "[[-0.02396103 -0.05518523 -0.08627115 -0.07779825 -0.02273003 -0.07515369\n",
      "  -0.06405259 -0.08325803 -0.04428619]]\n",
      "action 4\n",
      "0 False\n",
      "\r",
      " 0 0.0\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):    \n",
    "    print(\"\\n\", env.state)\n",
    "    \n",
    "    a_dst = sess.run(bot.predict_Q, {bot.input_s: [s]})\n",
    "    print(a_dst)\n",
    "    \n",
    "    for j in range(8):\n",
    "        a = np.argmax(a_dst)\n",
    "        if a not in env.valid_actions:\n",
    "            print(f\"Needed a {j}th try\")\n",
    "            a_dst[0][a] = - 10\n",
    "        else:\n",
    "            break\n",
    "\n",
    "    \n",
    "    s, r, d = env.step(a)\n",
    "\n",
    "    print(\"action\", a)\n",
    "    print(r, d)\n",
    "    total_r += r\n",
    "    \n",
    "    if d == True:\n",
    "        env.ss.show_bending_moment()\n",
    "        break\n",
    "\n",
    "print(\"\\r\", total_r, env.structure())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fab79d29240>]"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6UAAAFpCAYAAABklI6gAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3XdU1ufB//HPdTMFAUVARESWQtzb\nqFExS03b7CaapBmNJmqaJh1ZT58nfZqmTzPaZjVTs4dpdtJmmKRx771RWSIiAirIkHl/f3/E5pda\nE1GB6x7v1zk5J8C38M7pOcCH676/t3EcRwAAAAAA2OCyHQAAAAAA8F+MUgAAAACANYxSAAAAAIA1\njFIAAAAAgDWMUgAAAACANYxSAAAAAIA1jFIAAAAAgDWMUgAAAACANYxSAAAAAIA1jFIAAAAAgDWB\ntr5wTEyMk5ycbOvLAwAAAADa0Nq1a8sdx4k90XXWRmlycrLWrFlj68sDAAAAANqQMWZ3S67j4bsA\nAAAAAGsYpQAAAAAAaxilAAAAAABrGKUAAAAAAGsYpQAAAAAAaxilAAAAAABrGKUAAAAAAGsYpQAA\nAAAAaxilAAAAAABrTjhKjTEvGGNKjTFbvuPjxhjzuDEmxxizyRgzpPUzAQAAAAC+qCUnpS9JmvQ9\nH58sqdfRf26S9PTpZwEAAAAA/MEJR6njOIskHfyeSy6S9IrztRWSOhljurVWIADArttvv1233367\n7QwAAOCjAlvhc3SXtOdbbxcdfd++Yy80xtykr09TlZSU1ApfGgDQ1jZs2GA7AQAA+LDWGKXmOO9z\njneh4zjPSXpOkoYNG3bcawAA8CaO4+i1lYX6ZNM+Ocf/8dciLmN08eDu+vHQRBlzvB+tAAD4pta4\n+26RpB7fejtRUnErfF4AADzawZoGTX9ljf7ngy06VNsgt6NT/qe0ql53vrNJt85dr8ojjbb/0wAA\naDetcVL6kaSfGWPelDRSUqXjOP/x0F0AAHzJstxy/eJvG3SoplH3/rCPbhiTfFonnM1uR88szNVf\nvtipDXsq9NiUwRras3MrFgMA4Jla8pIwcyUtl5RhjCkyxtxojJlhjJlx9JJPJOVJypE0W9KsNqsF\nAMCyxma3/jRvh66es1LhIYF6/5bR+ulZKaf9kNsAl9EtE9L19oxRkqQrnl2uv361S81unu0CAPBt\nJzwpdRxn6gk+7ki6pdWKAADwUHsO1uq2N9drXWGFrhzWQ7+9sI/CglvjQUf/35CkzvrktrH6zftb\n9KfPd2ppzgE9cuUgxUeFturXAQDAU7TGc0oBAPB5f99YrAseW6xd+6v1xNTBevDyAa0+SP8lMjRI\nj08ZpIcvH6ANeyo0+bFF+nLb/jb5WgAA2MYoBQDge9Q2NOnOdzbq1rnrld61oz65bax+NDChzb+u\nMUY/HtZD//j5WeoW1UHTXlmj3364RXWNzW3+tQEAaE9t8ydeAAB8wNbiSt06d73yy2v0swnpuu3c\nXgoKaN+/56bFdtT7t4zWg5/u0AtL87Uy/6D+etVgpcdFtGsHAABthZNSAACO4TiOXliSr0ueXKaa\n+ia9Pm2kfj0xo90H6b+EBAbo3h/10YvXD1dZVb1++MQSzV1VqK9v6wAAgHdjlAIA8C0Hqut148tr\ndN8/tmlc7xh9ets4jU6LsZ0lSZqQGadPbxurYT2jdc97m3XLG+tUWctrmgIAvBujFACAo5bmlGvy\nY4u1JKdcv7uwr2ZfO0zR4cG2s/5NXGSoXvnpCN09OVOfb92vCx5frDUFB21nAQBwyhilAAC/19js\n1oOfZeua51cqIjRQH8wao+tGJ5/2a4+2FZfLaMb4NL0zc7QCXEZXPLtcj/+T1zQFAHgnRikAwK9V\n1TXqimeX6+kFuZoyvIf+futZ6pMQaTurRQb16KSPf36WfjQwQX/5YqeumbOSu/MCALwOoxQA4Lcc\nx9F/f7BFG/dU6Impg/XHS9vutUfbSkRokB69cpAeumyAlucd0O//sc12EgAAJ4VRCgDwW++sLdKH\nG4r1i3N7t8trj7YVY4yuGN5DN49L1esrC/Xp5n22kwAAaDFGKQDAL+WUVuveD7fqzNRozZqQbjun\nVfzq/AwNTIzSXe9uUtGhWts5AAC0CKMUAOB36hqbdevc9eoQHKDHpgxWgMszb2h0soIDXXpi6hA5\njvTzuevV2Oy2nQQAwAkxSgEAfuePn2zX9n2H9acfD1DXyFDbOa0qqUuY/nBpf60rrNCjX+60nQMA\nwAkxSgEAfuXzrSV6eflu3XhWis7O7Go7p01cODBBVw7roacW5GppTrntHAAAvhejFADgN4orjuiO\ndzapX/dI3Tkpw3ZOm/rthX2UGhOu2/+2QeXV9bZzAAD4ToxSAIBfaGp26/Y3N6ip2a0npg5RSGCA\n7aQ2FRYcqL9eNUSVRxr167c3yu12bCcBAHBcjFIAgF94/KscrSo4qPsv6aeUmHDbOe3ijG6R+p8f\nnKEFO8r0/JJ82zkAABwXoxQA4POW5x7QX7/apcuGJOqSwYm2c9rVNWf21MS+XfXQvGxt3FNhOwcA\ngP/AKAUA+LSDNQ26/W/rldwlXPdd1Nd2TrszxujBywYotmOIbp27XlV1jbaTAAD4N4xSAIDPchxH\nd7y9UYdqGvX41MEKDwm0nWRFp7BgPT51sPZWHNFv3t8ix+H5pQAAz8EoBQD4rBeXFuif2aW654JM\n9eseZTvHqmHJ0br9nF76aGOx3l5bZDsHAIBvMEoBAD5py95KPfBpts49I07Xj062neMRZk1I16jU\nLvrth1uVU1ptOwcAAEmMUgCAD6qub9Ktc9crOjxYD18+UMYY20keIcBl9OiUQeoQHKCfvbFOdY3N\ntpMAAGCUAgB8z70fbtHuAzV6dMogdQ4Ptp3jUbpGhupPPx6g7JIq/fGT7bZzAABglAIAfMt764r0\n3rq9uvXsXjoztYvtHI90dmZX3XhWil5evlvztpbYzgEA+DlGKQDAZ+SVVeu/P9iiESnRuvXsdNs5\nHu3OSRnq3z1Kd76zScUVR2znAAD8GKMUAOAT6puadevc9QoOdOmxKYMUGMCPuO8TEhigJ6YOVlOz\nW7e9uV5NzW7bSQAAP8VPbACAT3jw0x3aWnxYD18+UN2iOtjO8QrJMeH6wyX9tbrgkB7/Ksd2DgDA\nTzFKAQBe75/b9+uFpfm6fnSyzuvT1XaOV7l4cHddNiRRT3y1S8tzD9jOAQD4IUYpAMCrlVTW6ddv\nb1SfbpG6e3Km7RyvdN9FfZXSJVy3/229DtY02M4BAPgZRikAwKv91/ubVd/k1hNXDVZoUIDtHK8U\nHhKoJ64arEM1jfrtR1tt5wAA/AyjFADgtZblluur7FLddk4vpcV2tJ3j1fomROnm8an6+8ZibSqq\nsJ0DAPAjjFIAgFdyHEcPfpqthKhQXTc62XaOT7hpXKqiw4P1wKfZchzHdg4AwE8wSgEAXunjzfu0\nsahSvzivNw/bbSURoUG69ex0Lcs9oIU7y2znAAD8BKMUAOB1GpvdenjeDmXGR+jSIYm2c3zK1SN7\nKik6TA98mi23m9NSAEDbY5QCALzO3FWF2n2gVndNylSAy9jO8SnBgS79emKGskuq9MGGvbZzAAB+\ngFEKAPAq1fVNeuzLXRqZEq2sjFjbOT7ph/27qX/3KP35852qa2y2nQMA8HGMUgCAV5m9KE8Hahp0\nzwVnyBhOSduCy2V09+RM7a04otdW7LadAwDwcYxSAIDXKK2q0+zFefpB/24a1KOT7RyfNiY9RuN6\nx+qv83NUeaTRdg4AwIcxSgEAXuPxf+5SQ5Nbv56YYTvFL9w9KVOVRxr19IJc2ykAAB/GKAUAeIW8\nsmrNXbVHU0ckKSUm3HaOX+iTEKmLB3XXi0vzta/yiO0cAICPYpQCALzCnz7foZBAl35+Ti/bKX7l\nl+f1luNIj3yx03YKAMBHMUoBAB5vfeEhfbK5RNPHpio2IsR2jl/pER2ma0f11Dtri7Rzf5XtHACA\nD2KUAgA8muM4+uOn2YrpGKzp41Jt5/ilWyakKzwkUA99lm07BQDggxilAACPNn9HqVblH9Rt5/RS\nx5BA2zl+qXN4sGZmpenL7V//fwEAQGtilAIAPFaz29GDn+5QcpcwTRmRZDvHr/10TIriI0P1x0+3\ny3Ec2zkAAB/CKAUAeKz31hVpx/4q3TExU0EB/MiyKTQoQL84r5fWF1Zo3tYS2zkAAB/CT3gAgEeq\na2zWX77YqYE9OumC/vG2cyDpsiGJ6hXXUQ99tkONzW7bOQAAH8EoBQB4pJeWFWhfZZ3unpQpY4zt\nHEgKDHDprkmZyiuv0d9W77GdAwDwEYxSAIDHqaht0FPzczQhI1aj0rrYzsG3nHNGnEYkR+vRL3ep\npr7Jdg4AwAe0aJQaYyYZY3YYY3KMMXcf5+NJxpj5xpj1xphNxpgLWj8VAOAvnlqQq6r6Jt01OdN2\nCo5hjNFdkzNVXl2v55fk284BAPiAE45SY0yApCclTZbUR9JUY0yfYy77b0lvOY4zWNIUSU+1digA\nwD/srTiil5YV6NLBicqMj7Sdg+MY2rOzJvWN17MLc1VeXW87BwDg5VpyUjpCUo7jOHmO4zRIelPS\nRcdc40j6128OUZKKWy8RAOBP/vL5TknSL8/vbbkE3+eOSRmqa3Lrr1/l2E4BAHi5lozS7pK+fTeD\noqPv+7b/lXSNMaZI0ieSbm2VOgCAX9m+77DeW1+kG0Ynq3unDrZz8D3SYjvqyuE99PrK3dp9oMZ2\nDgDAi7VklB7vlofHvmr2VEkvOY6TKOkCSa8aY/7jcxtjbjLGrDHGrCkrKzv5WgCAT3vws2xFhARq\nZlaa7RS0wO3n9FKgy6WH5+2wnQIA8GItGaVFknp86+1E/efDc2+U9JYkOY6zXFKopJhjP5HjOM85\njjPMcZxhsbGxp1YMAPBJy3LLtWBHmW6ZkK5OYcG2c9ACcZGhmj42Rf/YtE+biips5wAAvFRLRulq\nSb2MMSnGmGB9fSOjj465plDSOZJkjDlDX49SjkIBAC3iOI4e/DRbCVGhum50su0cnITp41IVHR6s\nBz7NluMc+0AqAABO7ISj1HGcJkk/kzRP0nZ9fZfdrcaY+4wxFx697FeSphtjNkqaK+l6h59MAIAW\n+njzPm0sqtQvzuut0KAA2zk4CRGhQfr52elalntAC3fy92gAwMkLbMlFjuN8oq9vYPTt9937rX/f\nJmlM66YBAPxBY7NbD8/bocz4CF06JNF2Dk7BVSN76sVlBXrg02yN6xUrl+t4t6MAAOD4WvLwXQAA\n2szcVYXafaBWd03KVABjxisFB7r06/MzlF1SpQ827LWdAwDwMoxSAIA11fVNevyfuzQyJVpZGdwA\nz5v9oH839e8epT9/vlN1jc22cwAAXoRRCgCw5vUVu1Ve3aC7J2fKGE5JvZnLZXTXpEztrTiid9cV\n2c4BAHgRRikAwIq6xmbNWZKvs9JjNDips+0ctIIx6V00qEcnPbswT03Nbts5AAAvwSgFAFjx/vq9\nKquq18ysNNspaCXGGM3MSlPhwVp9sqXEdg4AwEswSgEA7a7Z7ejZhbkakBil0WldbOegFZ13Rlel\nxYbr6QW5vG4pAKBFGKUAgHb36ZZ9KjhQq5nj03guqY9xuYxmjE/T9n2Hed1SAECLMEoBAO3KcRw9\nvSBXqbHhmtg33nYO2sBFg7orISpUTy/ItZ0CAPACjFIAQLtavKtcW4sPa8a4NLl4XVKfFBzo0rSx\nqVqZf1Brdx+ynQMA8HCMUgBAu3pqQY7iI0N18eDutlPQhqaM6KFOYUGclgIATohRCgBoN+sKD2lF\n3kFNG5ui4EB+BPmysOBAXT86WV9u36+d+6ts5wAAPBi/EQAA2s0zC3IV1SFIU0ck2U5BO7huVLLC\nggP0DKelAIDvwSgFALSLXfur9Pm2/bpudLLCQwJt56AddA4P1tQRSfpwY7GKDtXazgEAeChGKQCg\nXTyzME8dggJ0/ehk2yloR9PGpshlpDmL822nAAA8FKMUANDm9lYc0Ycb9mrKiB6KDg+2nYN21C2q\ngy4e1F1vri7Ugep62zkAAA/EKAUAtLk5i/MkSdPGplougQ03j09TfZNbLy0rsJ0CAPBAjFIAQJs6\nWNOgN1ft0UWDuqt7pw62c2BBelxHTewTr5eXFai6vsl2DgDAwzBKAQBt6qVlBTrS2KyZWZyS+rMZ\nWWk6XNekuSsLbacAADwMoxQA0GZq6pv08rICnd+nq9LjImznwKJBPTppdFoXzVmSp/qmZts5AAAP\nwigFALSZuasKVXmkUTOy0mynwAPMykrX/sP1en/dXtspAAAPwigFALSJ+qZmzVmcrzNTozUkqbPt\nHHiAMeld1L97lJ5dlKdmt2M7BwDgIRilAIA28eH6YpUcrtOsrHTbKfAQxhjNzEpTfnmN5m0tsZ0D\nAPAQjFIAQKtrdjt6ZlGu+iZEamyvGNs58CAT+8YrNSZcTy3IkeNwWgoAYJQCANrAF9tKlFdWo5lZ\naTLG2M6BBwlwGd08PlVb9h7Wkpxy2zkAAA/AKAUAtCrHcfTUglwldwnT5H7dbOfAA108uLu6Robo\n6QW5tlMAAB6AUQoAaFXLcg9oU1Glbh6fpgAXp6T4TyGBAZp2VqqW5R7Qhj0VtnMAAJYxSgEArerp\nBbmKiwjRpUO6206BB5s6MklRHYL09IIc2ykAAMsYpQCAVrOpqEJLcsp141kpCgkMsJ0DD9YxJFDX\njeqpeVv3K6e0ynYOAMAiRikAoNU8vSBXkaGBumpkku0UeIHrRicrNMilZxbm2U4BAFjEKAUAtIrc\nsmp9trVE145KVkRokO0ceIEuHUM0ZXiSPli/V8UVR2znAAAsYZQCAFrFcwvzFBzg0vVjkm2nwItM\nG5siSZqzON9yCQDAFkYpAOC07as8ovfWF2nK8B6K6RhiOwdeJLFzmC4clKC5qwp1qKbBdg4AwAJG\nKQDgtD2/OF9uR5o2NtV2CrzQjPFpOtLYrJeWFdhOAQBYwCgFAJyWitoGvbGqUBcOTFCP6DDbOfBC\nvbtG6Nwzuurl5QWqqW+ynQMAaGeMUgDAaXll+W7VNjRrxvg02ynwYjOz0lRR26g3V++xnQIAaGeM\nUgDAKattaNKLS/N17hlxyoiPsJ0DLza0Z2eNTInWnMV5amhy284BALQjRikA4JT9bfUeHapt1Mws\nTklx+mZmpWlfZZ0+2LDXdgoAoB0xSgEAp6Sx2a05i/M1PLmzhvaMtp0DHzC+d6z6dIvUswtz5XY7\ntnMAAO2EUQoAOCWfbN6nvRVHeC4pWo0xRjePT1VuWY3m7yi1nQMAaCeMUgDASXMcR7MX5yktNlwT\nMuJs58CHXNC/mxKiQvXcojzbKQCAdsIoBQCctOV5B7Rl72FNG5sql8vYzoEPCQpw6adnpWhl/kFt\nKqqwnQMAaAeMUgDASZu9KE8xHYN1yeDutlPgg64c3kMRIYGavTjfdgoAoB0wSgEAJ2Xn/irN31Gm\na0clKzQowHYOfFBEaJCuGpmkTzbv056DtbZzAABtjFEKADgpcxbnKTTIpWvO7Gk7BT7s+jHJMpJe\nXFpgOwUA0MYYpQCAFis9XKcP1hfrx0N7KDo82HYOfFi3qA66cGCC3lxdqMraRts5AIA2xCgFALTY\ny8sL1Oh268azUmynwA9MG5uq2oZmvbGq0HYKAKANMUoBAC1SU9+k11YUamKfeCXHhNvOgR/okxCp\nsb1i9OLSfDU0uW3nAADaCKMUANAib6/Zo8ojjZo+LtV2CvzI9LGpKq2q10cbi22nAADaCKMUAHBC\njiM9vzRfQ3t21tCenW3nwI+M7RWjzPgIzV6UJ8dxbOcAANoAoxQAcEIHaxu05+ARTR/LKSnalzFG\n08emasf+Ki3aVW47BwDQBlo0So0xk4wxO4wxOcaYu7/jmiuMMduMMVuNMW+0biYAwKZ9FUeU3CVM\n5/XpajsFfuhHAxPUNTJEsxfl2U4BALSBE45SY0yApCclTZbUR9JUY0yfY67pJekeSWMcx+kr6fY2\naAUAWFBV16Tq+ibdODZVAS5jOwd+KDjQpRvGpGhJTrm2FlfazgEAtLKWnJSOkJTjOE6e4zgNkt6U\ndNEx10yX9KTjOIckyXGc0tbNBADYUlxxRIEBLl0+JNF2CvzY1BFJCg8O0JzF+bZTAACtrCWjtLuk\nPd96u+jo+76tt6TexpilxpgVxphJrRUIALAnt6xah2obFB8Zqg7BAbZz4MeiOgTpyuFJ+vvGYhVX\nHLGdAwBoRS0Zpcd7rNaxt78LlNRLUpakqZLmGGM6/ccnMuYmY8waY8yasrKyk20FALSzOYvz5TJG\nXSNDbKcAumFMshxJLy0rsJ0CAGhFLRmlRZJ6fOvtREnHvlhYkaQPHcdpdBwnX9IOfT1S/43jOM85\njjPMcZxhsbGxp9oMAGgH5dX1enddkWIiQhQUwM3aYV+P6DBd0L+b3lhZqMN1jbZzAACtpCW/ZayW\n1MsYk2KMCZY0RdJHx1zzgaQJkmSMidHXD+flFnkA4MVeWb5bDU1udYsKtZ0CfGP62BRV1zfpb6v2\nnPhiAIBXOOEodRynSdLPJM2TtF3SW47jbDXG3GeMufDoZfMkHTDGbJM0X9IdjuMcaKtoAEDbOtLQ\nrFeXF+jcM7qqQxDPJYXnGJDYSWemRuuFpflqbHbbzgEAtIIWPR7LcZxPHMfp7ThOmuM4fzj6vnsd\nx/no6L87juP80nGcPo7j9Hcc5822jAYAtK131hXpUG2jbhqXajsF+A83jUvVvso6fbxpn+0UAEAr\n4ElCAIB/0+x29PziPA3s0UnDkzvbzgH+Q1bvOKXHddRzi/LkOMfeexEA4G0YpQCAf/PFtv0qOFCr\nm8amypjj3YAdsMvlMpo+NkXb9h3WslyeLQQA3o5RCgD4N7MX56lHdAdN7NvVdgrwnS4a1F0xHUP0\n3CLuqwgA3o5RCgD4xtrdB7V29yHdOCZFgbwMDDxYaFCArh/dUwt3lmlHSZXtHADAaeA3DgDAN2Yv\nyldUhyD9eFiPE18MWHb1yJ7qEBSg2Ys5LQUAb8YoBQBIkgrKazRvW4muOTNJ4SGBtnOAE+ocHqwr\nhiXqww17tf9wne0cAMApYpQCACRJzy/JV5DLpetGJdtOAVrsp2elqNnt6KVlBbZTAACniFEKANDB\nmga9vXaPLh6coLjIUNs5QIv17BKuSf3i9fqK3aqub7KdAwA4BYxSAIBeW7FbdY1uTRubajsFOGnT\nx6bqcF2T3lq9x3YKAOAUMEoBwM/VNTbr5WUFmpARq95dI2znACdtcFJnDU/urOeX5Kup2W07BwBw\nkhilAODn3l+/VwdqGjR9HKek8F7Tx6Zqb8URfbqlxHYKAOAkMUoBwI+53Y5mL85T34RIjUrtYjsH\nOGXnntFVKTHhem5RnhzHsZ0DADgJjFIA8GNfZZcqr6xGN41LlTHGdg5wylwuo2ljU7R5b6VW5h+0\nnQMAOAmMUgDwY88tzlNCVKgu6N/Ndgpw2i4bkqjo8GDNXpRnOwUAcBIYpQDgpzbuqdCq/IP66Vkp\nCgrgxwG8X2hQgK4d1VP/zC5VTmmV7RwAQAvxWwgA+KmnF+QqIjRQVw7vYTsFaDU/ObOnQoNcemYh\np6UA4C0YpQDgh3JKq/TZ1hJdNypZEaFBtnOAVtOlY4imDE/SB+v3am/FEds5AIAWYJQCgB96ekGe\nQoNcumFMsu0UoNXddPTljXhuKQB4B0YpAPiZokO1+nDDXk0ZnqQuHUNs5wCtLqFTB10yuLvmripU\neXW97RwAwAkwSgHAz/zr9Ohfp0mAL5qRlaaGZrdeXJpvOwUAcAKMUgDwI+XV9Xpz9R5dMri7Ejp1\nsJ0DtJm02I6a3C9eryzfrcN1jbZzAADfg1EKAH7khSX5amh2a0ZWmu0UoM3NykpXVV2TXlux23YK\nAOB7MEoBwE8crmvUq8t3a3K/eKXFdrSdA7S5ft2jNK53rF5Ykq+6xmbbOQCA78AoBQA/8dqK3aqq\nb9KsrHTbKUC7mZWVpvLqBr21Zo/tFADAd2CUAoAfqGts1gtL8jWud6z6dY+ynQO0m5Ep0Rras7Oe\nXZinxma37RwAwHEwSgHAD7y1Zo/Kqxs0i+eSws8YYzQrK017K47o7xuLbecAAI6DUQoAPq6x2a1n\nF+ZpaM/OGpkSbTsHaHdnZ8YpMz5CTy3Ildvt2M4BAByDUQoAPu6jDcXaW3FEs7LSZIyxnQO0O2OM\nZmalKae0Wl9s3287BwBwDEYpAPgwt9vR0wtzlRkfobMz42znANb8oH83JUWH6an5OXIcTksBwJMw\nSgHAh32+bb9ySqs1k1NS+LnAAJdmjE/TxqJKLcs9YDsHAPAtjFIA8FGO4+jpBTlKig7TD/p3s50D\nWHfZ0O6KiwjRk/NzbKcAAL6FUQoAPmppzgFtLKrUjPFpCgzg2z0QEhig6WNTtSz3gDbsqbCdAwA4\nit9SAMBHPbUgR3ERIbpsaHfbKYDHmDoySVEdgvQUp6UA4DEYpQDgg9YXHtKy3AOaPjZVIYEBtnMA\nj9ExJFDXj07W59v2a+f+Kts5AAAxSgHAJz21IFdRHYI0dWSS7RTA41w/OllhwQF6ZkGu7RQAgBil\nAOBzdu6v0hfb9uu60cnqGBJoOwfwOJ3DgzV1RJI+3FisPQdrbecAgN9jlAKAj3l6Qa7CggN0w+hk\n2ymAx5o+NlUuIz23KM92CgD4PUYpAPiQPQdr9dHGYk0dkaTO4cG2cwCPFR8VqsuGJOpva/aotKrO\ndg4A+DVGKQD4kOcW5cllvj4FAvD9bh6fpqZmt15YUmA7BQD8GqMUAHxEaVWd/rZmjy4bkqj4qFDb\nOYDHS4kJ1wX9u+m1FbtVeaTRdg4A+C1GKQD4iBeWFKip2a2bx6fZTgG8xqysdFXXN+nV5QW2UwDA\nbzFKAcAHVB5p1GsrduuC/t2UEhNuOwfwGn0SIjUhI1YvLC3QkYZm2zkA4JcYpQDgA15dXqDq+ibN\nzOKUFDhZsyak62BNg95cXWg7BQD8EqMUALzckYZmvbC0QBMyYtU3Icp2DuB1hidHa0RytGYvylND\nk9t2DgD4HUYpAHi5N1cX6mDmZ9Z4AAAgAElEQVRNg2ZNSLedAnitmRPSVFxZpw827LWdAgB+h1EK\nAF6socmt2YvyNCI5WsOTo23nAF4rq3es+nSL1DMLc9XsdmznAIBfYZQCgBf7cMNeFVfWaeYEnksK\nnA5jjGZNSFNeWY0+31piOwcA/AqjFAC8VLPb0dMLc9WnW6SyesfazgG83uR+X9+9+skFOXIcTksB\noL0wSgHAS32+tUR5ZTWaNSFNxhjbOYDXC3AZzRifqi17D2vxrnLbOQDgNxilAOCFHMfRkwtylBIT\nrsn9utnOAXzGJYMT1S0qVH+dz2kpALSXFo1SY8wkY8wOY0yOMebu77nucmOMY4wZ1nqJAIBjfbal\nRFv2HtasrDQFuDglBVpLcKBLM8anaVX+QU5LAaCdnHCUGmMCJD0pabKkPpKmGmP6HOe6CEk/l7Sy\ntSMBAP9fU7NbD3++Q73iOurSIYm2cwCfM2VEDyV27qCH5mXLzZ14AaDNteSkdISkHMdx8hzHaZD0\npqSLjnPd7yU9JKmuFfsAAMd4Z22R8spqdMfEDE5JgTYQEhigX53fW1v2HtYnW/bZzgEAn9eSUdpd\n0p5vvV109H3fMMYMltTDcZx/tGIbAOAYdY3NevTLXRqc1Enn9elqOwfwWRcO7K7M+Aj9ad4ONTa7\nbecAgE9rySg93p/hv3ksizHGJekRSb864Scy5iZjzBpjzJqysrKWVwIAJEmvLC9QyeE63TUpkzvu\nAm0owGV0x8QMFRyo1Vtr9pz4fwAAOGUtGaVFknp86+1EScXfejtCUj9JC4wxBZLOlPTR8W525DjO\nc47jDHMcZ1hsLK+pBwAno/JIo56cn6usjFidmdrFdg7g887OjNPw5M567MtdOtLQbDsHAHxWS0bp\nakm9jDEpxphgSVMkffSvDzqOU+k4TozjOMmO4yRLWiHpQsdx1rRJMQD4qecW5arySKPumJhhOwXw\nC8YY3TkpU6VV9XpxWb7tHADwWSccpY7jNEn6maR5krZLestxnK3GmPuMMRe2dSAAQCo9XKcXlhTo\nwoEJ6psQZTsH8BvDk6N1TmacnlmQq8raRts5AOCTWvQ6pY7jfOI4Tm/HcdIcx/nD0ffd6zjOR8e5\nNotTUgBoXU98laPGZrd+eV5v2ymA37ljUoaq6pv09MJc2ykA4JNaNEoBAPYUlNdo7qpCTR2RpOSY\ncNs5gN/JjI/UJYO668Wl+Sqp5JXvAKC1MUoBwMP95YudCgpw6daz022nAH7rF+f1lttx9Ng/d9lO\nAQCfwygFAA+2tbhSH20s1k/PSlZcZKjtHMBv9YgO09Uje+qtNXuUV1ZtOwcAfAqjFAA82EOf7VCn\nsCDdPD7Ndgrg9352drpCAl368xc7bacAgE9hlAKAh1qee0ALd5ZpVlaaIkODbOcAfi+mY4imjU3V\nx5v2aXNRpe0cAPAZjFIA8ECO4+ihednqFhWqa0cl284BcNT0sSmKDg/WQ/OybacAgM9glAKAB/p8\n236tL6zQ7ef2UmhQgO0cAEdFhAbplgnpWryrXEtzym3nAIBPYJQCgIdpdjt6eN4OpcaG67IhibZz\nABzj6pFJ6t6pgx76LFuO49jOAQCvxygFAA/z3roi5ZRW647zMxQYwLdpwNOEBgXo9nN7aWNRpT7b\nUmI7BwC8Hr/tAIAHqWts1iNf7NTAxChN6hdvOwfAd7h0SKJ6xXXUw5/vUFOz23YOAHg1RikAeJDX\nVuxWcWWd7pqUKWOM7RwA3yHAZXTHxAzlldXo3XVFtnMAwKsxSgHAQ1TVNerJ+Tka2ytGo9NjbOcA\nOIHz+nTVkKROeuSLXaprbLadAwBei1EKAB5i9qI8Hapt1J0TM22nAGgBY4zumpSpksN1emV5ge0c\nAPBajFIA8ABlVfWasyRfPxjQTf0To2znAGihkaldlJURqyfn56rySKPtHADwSoxSAPAAT87PUX2T\nW786r7ftFAAn6Y6JGao80qjZi/JspwCAV2KUAoBlhQdq9frK3bpyeA+lxna0nQPgJPVNiNJFgxL0\n/JJ8lR6us50DAF6HUQoAlj3y5U65jNFt5/SynQLgFP3yvN5qbHbria9ybKcAgNdhlAKARdv3HdYH\nG/bqhjEp6hoZajsHwCnq2SVcU0ckae6qQu0+UGM7BwC8CqMUACx6eN4ORYQEaub4NNspAE7Treek\nKyjApT9/vtN2CgB4FUYpAFiyKv+gvsou1cysdEWFBdnOAXCa4iJCdeNZKfpoY7G2FlfazgEAr8Eo\nBQALHMfRQ59lKy4iRNePTradA6CV3DQ+VZ3CgvTwvB22UwDAazBKAcCCL7eXas3uQ7rt3F7qEBxg\nOwdAK4kMDdKsrDQt2FGm5bkHbOcAgFdglAJAO6ttaNLv/r5V6XEddcWwHrZzALSya0clK7FzB/3P\nh1tU39RsOwcAPB6jFADa2V8+36miQ0f0x0v7KyiAb8OArwkNCtD9F/dTTmm1nl6QazsHADwevw0B\nQDvaVFShF5bm66qRSRqeHG07B0AbycqI00WDEvTU/FzllFbZzgEAj8YoBYB20tjs1t3vblZMxxDd\nPTnTdg6ANvY/P+yjsJAA3f3uZrndju0cAPBYjFIAaCfPL8nXtn2Hdd9FfRUZykvAAL4upmOI/vsH\nfbRm9yG9sarQdg4AeCxGKQC0g90HavTIFzt1fp+umtSvm+0cAO3ksiHddVZ6jB78NFsllXW2cwDA\nIzFKAaCNOY6j37y/RcEBLt13UT/bOQDakTFGf7iknxqa3frfj7bazgEAj8QoBYA29t66vVqSU647\nJ2cqPirUdg6AdtazS7huP7e3PttaonlbS2znAIDHYZQCQBs6UF2v+z/epqE9O+vqEUm2cwBYMm1s\nis7oFql7P9yiw3WNtnMAwKMwSgGgDf3+H9tUXd+kBy7tL5fL2M4BYElQgEsPXtZfZVX1euizbNs5\nAOBRGKUA0EYW7izTBxuKNTMrXb26RtjOAWDZgMROumFMil5bUag1BQdt5wCAx2CUAkAbqG1o0m/e\n36y02HDdMiHNdg4AD/HL83qre6cOuvu9zapvaradAwAegVEKAG3gL5/vVNGhI3rgsgEKCQywnQPA\nQ4SHBOr+S/opp7RaTy/ItZ0DAB6BUQoArWxzUaVeWJqvq0YmaXhytO0cAB5mQkacLhyYoKfm5yqn\ntMp2DgBYxygFgFbU1OzW3e9tUkzHEN09OdN2DgAPde+P+igsJED3vLdZbrdjOwcArGKUAkAren5J\nvrYWH9Z9F/VVZGiQ7RwAHiqmY4h+c8EZWl1wSHNXF9rOAQCrGKUA0EoKD9TqkS936rw+XTWxb7zt\nHAAe7vKhiRqT3kUPfJKt/YfrbOcAgDWMUgBoBY7j6L/e36xAl0u/v6ifjOE1SQF8P2OM/nBxfzU0\nu/XbD7fazgEAaxilANAK3lu3V0tyynXXpAzFR4XazgHgJZJjwnXbub302dYSzdtaYjsHAKxglALA\naTpQXa/7P96moT076+qRPW3nAPAy08em6oxukbr3wy06XNdoOwcA2h2jFABO0/0fb1d1fZMeuLS/\nXC4etgvg5AQFuPTApf1VVlWvhz/bYTsHANodoxQATsPCnWV6f/1ezcxKV6+uEbZzAHipgT066frR\nKXpt5W6t3X3Qdg4AtCtGKQCcotqGJv3m/c1KjQ3XrKw02zkAvNyvzu+thKgOuvvdzapvaradAwDt\nhlEKAKfokS92qujQET1w6QCFBgXYzgHg5cJDAnX/xf20q7RazyzIs50DAO2GUQoAp2BzUaWeX5Kv\nqSOSNCIl2nYOAB8xITNOPxqYoCfn5yintMp2DgC0C0YpAJykhia37n5vk2I6hujuyZm2cwD4mHt/\n2EcdggN0z3ub1ex2bOcAQJtjlALASfrd37dqa/Fh/f7iforqEGQ7B4CPiY0I0W9/1EerCw7poXnZ\ntnMAoM0F2g4AAG/yxspCvb6yUDOz0jSxb7ztHAA+6tIhiVpfWKFnF+apb0KULhyYYDsJANoMJ6UA\n0EJrdx/Ubz/aovG9Y/Xr8zNs5wDwcf/zwz4akRytO9/ZqK3FlbZzAKDNtGiUGmMmGWN2GGNyjDF3\nH+fjvzTGbDPGbDLG/NMY07P1UwHAnpLKOs14bZ26d+qgx6cMVoDL2E4C4OOCA1168uoh6hwWrJte\nWauDNQ22kwCgTZxwlBpjAiQ9KWmypD6Sphpj+hxz2XpJwxzHGSDpHUkPtXYoANhS19ism19bq9r6\nJj137TBFhfE8UgDtIzYiRM/+ZKjKqut1y+vr1NTstp0EAK2uJSelIyTlOI6T5zhOg6Q3JV307Qsc\nx5nvOE7t0TdXSEps3UwAsMNxHN374RZt3FOhP18xUL27RthOAuBnBiR20h8v6a/leQf0f59w4yMA\nvqclo7S7pD3fervo6Pu+y42SPj2dKADwFK+u2K231hTp52ena1K/brZzAPipy4Ym6qdjUvTC0ny9\nu7bIdg4AtKqWjNLjPXHquC+aZYy5RtIwSQ9/x8dvMsasMcasKSsra3klAFiwIu+A7vv7Np2TGafb\nz+1tOweAn/uvCzI1Oq2L7nl/szYVVdjOAYBW05JRWiSpx7feTpRUfOxFxphzJf1G0oWO49Qf7xM5\njvOc4zjDHMcZFhsbeyq9ANAu9lYc0S2vr1NSlzA9MmWQXNzYCIBlgQEu/fWqIYrtGKKbX12rsqrj\n/roFAF6nJaN0taRexpgUY0ywpCmSPvr2BcaYwZKe1deDtLT1MwGg/dQ1NuvmV9eoocmt2dcOU2Qo\nNzYC4Bmiw4P13LVDdai2QbNeX6uGJm58BMD7nXCUOo7TJOlnkuZJ2i7pLcdxthpj7jPGXHj0socl\ndZT0tjFmgzHmo+/4dADg0RzH0T3vbdbW4sN6dMogpcV2tJ0EAP+mb0KUHrp8oFYXHNLv/7HNdg4A\nnLbAllzkOM4nkj455n33fuvfz23lLgCw4vkl+Xp//V796rzeOueMrrZzAOC4LhyYoK3FlXp2YZ76\nJkRqyogk20kAcMpa8vBdAPALS3aV6/8+2a5JfeN1y4R02zkA8L3unJipsb1idO+HW7V29yHbOQBw\nyhilACBpz8Fa/WzuOqXHddSfrhjIjY0AeLwAl9ETUwcrPipUM19bq/2H62wnAcApYZQC8Hu1DU2a\n/soaud2OnvvJMHUMadEzGwDAuk5hX9/4qLq+STNeW6v6pmbbSQBw0hilAPya4zi6451N2rm/Sk9c\nNUTJMeG2kwDgpGTGR+rPPx6o9YUVuveDrXKc476cPAB4LEYpAL/2zMI8fbxpn+6clKnxvXn9ZADe\naXL/bvrZhHT9bc0evbay0HYOAJwURikAv7VgR6kempetHw7oppvHpdrOAYDT8ovzeuvszDj97qOt\nWpV/0HYOALQYoxSAX8ovr9HP565XRtcIPXT5ABnDjY0AeLcAl9EjVw5SUnSYZr2+VsUVR2wnAUCL\nMEoB+J3q+ibd9MoauVxGs68dprBgbmwEwDdEdQjSc9cOVV2jWze/ulZ1jdz4CIDnY5QC8Cs19U2a\n8epa5ZZV68mrhqhHdJjtJABoVelxEXrkykHavLdSP3tjPcMUgMdjlALwGxW1Dbrm+ZValluuhy4f\nqDHpMbaTAKBNnNenq+67qK++3L5fN7y4WtX1TbaTAOA7MUoB+IXSw3W68tkV2rr3sJ66eqguH5po\nOwkA2tS1o5L1yJUDtargoK6evUKHahpsJwHAcTFKAfi8wgO1uvyZ5dpzqFYv3jBck/rF204CgHZx\nyeBEPXvNUG0vqdIVzy5XSWWd7SQA+A+MUgA+bUdJlS5/ZpkqjzTq9WkjecguAL9zbp+uevmGEdpX\nWafLnl6mgvIa20kA8G8YpQB81vrCQ7ri2eWSpLduHqXBSZ0tFwGAHaPSuuiN6SNV29Cky59Zrm3F\nh20nAcA3GKUAfNKSXeW6es5KRXUI0rszRysjPsJ2EgBYNSCxk96eMUpBAUZTnluutbsP2k4CAEmM\nUgA+6LMt+/TTl1YrKTpM78wYxcu+AMBR6XERenvGKHXpGKKr56zUwp1ltpMAgFEKwLe8tWaPZr2+\nTn27R+rNm85UXGSo7SQA8CiJncP01s2jlBrTUdNeXq1/bCq2nQTAzzFKAfiMOYvzdOc7mzQmPUav\nTxupTmHBtpMAwCPFRoRo7k1nalCPTrp17nrNXVVoOwmAH2OUAvB6juPoz5/v0P0fb9cF/eM157ph\nCgsOtJ0FAB4tqkOQXvnpSI3vHat73tusZxbm2k4C4KcYpQC8mtvt6H8/2qonvsrRlcN66ImpQxQS\nGGA7CwC8QofgAD33k2H64YBueuDTbD34WbYcx7GdBcDPcJQAwGs1Nrt1x9sb9cGGYt00LlX3TM6U\nMcZ2FgB4leBAlx6bMlhRHYL09IJcVR5p1O8v6qcAF99PAbQPRikAr1TX2KxbXl+nf2aX6o6JGZqV\nlcYgBYBTFOAyuv/iforqEKSnFuTq8JFG/eWKQQoO5EF1ANoeoxSA16mqa9S0l9doVcFB3X9xP11z\nZk/bSQDg9YwxunNSpqI6BOmPn2arur5JT189VB2CeUoEgLbFn78AeJU9B2s1dfYKrd19SI9NGcwg\nBYBWdvP4ND1waX8t2lmma55fqZLKOttJAHwcoxSAV3C7Hb2yvEATH12kgvJazb52mC4cmGA7CwB8\n0pQRSfrrVUO0rfiwzntkod5avYcbIAFoMzx8F4DH232gRne+s0kr8w9qXO9Y/fHS/ureqYPtLADw\naRf076a+CZG6851NuvPdTfrH5n164NL+SuD7L4BWxkkpAI/ldjt6YUm+Jj26WNuKD+uhywbo5RuG\nM0gBoJ307BKuudPP1O8v6qs1BQd1/iOLNHdVIaemAFoVJ6UAPFJ+eY3ufGejVhcc0oSMWP3fpf3V\nLYoxCgDtzeUy+smoZGVlxOnOdzbpnvc26+NN+/TAZf2V2DnMdh4AH8BJKQCP0ux2NGdxniY9ukg7\nSqr05x8P1AvXD2eQAoBlPaLD9Pq0kbr/4n5aX3hIEx9ZpFdX7JbbzakpgNPDSSkAj5FbVq073t6o\ndYUVOveMOP3hkv7qGhlqOwsAcJTLZXTNmT2VlRGru9/drP/5YIs+2bRPD10+QD2iOTUFcGo4KQVg\nXbPb0bMLczX5scXKLavRo1cO0uxrhzFIAcBDJXYO06s3jtADl/bX5r2VmvjoIr28rIBTUwCnhJNS\nAFbt2l+lX7+zSRv3VOj8Pl11/yX9FBfBGAUAT2eM0ZQRSRrXO1b3vLdZv/1oqz7evE8PXz5APbuE\n284D4EU4KQVgRVOzW0/Oz9EPHl+iwgM1enzqYD37k6EMUgDwMgmdOuilG4brocsHaPu+w5r46CK9\nsCSfU1MALcZJKYB2t6OkSne8s1Gbiip1Qf94/e7CfoqNCLGdBQA4RcYYXTGsh8b1itV/vb9Z9/1j\nmz7Z/PVzTVNjO9rOA+DhGKUA2k1pVZ1eWlqg2YvzFBkapCevGqIfDOhmOwsA0Erio0L1/HXD9N66\nvfrd37dq8mOLNTMrTdeOSlZ0eLDtPAAeilEKoM3tKKnS80vy9MH6YjW63bpwYILu/WEfdenI6SgA\n+BpjjC4bmqizesXotx9u1aNf7tIzC3N12ZBE3XhWCienAP4DoxRAm3AcR0tyyjV7cb4W7SxTaJBL\nVwxP1E/H8AsJAPiDrpGheuYnQ7Vzf5WeX5yvt9cU6Y1VhTonM07TxqZqZEq0jDG2MwF4AEYpgFZV\n39Ssv2/cpzmL85RdUqWYjiH69fm9dfXInurMQ7cAwO/07hqhBy8foF9PzNCrK3brtRW7NeW5Ferf\nPUrTxqbogv7dFBTAvTcBf2Ycx86d0YYNG+asWbPGytcG0Poqahv0+spCvbysQKVV9croGqEbx6bo\nokEJCgkMsJ2H05CVlSVJWrBggdUOAL6hrrFZ764r0vOL85VXXqOEqFBdPyZZU0YkKTI0yHYegFZk\njFnrOM6wE13HSSmA01JQXqMXln79sKwjjc0a2ytGf/rxQI3tFcPDsgAA/yE0KEBXj+ypqcOT9FV2\nqeYsydP/fZKtx/+ZoyuH99ANY5KV2DnMdiaAdsQoBXDSHMfRmt2HNGdxnj7ftl+BLqOLBnXXtLEp\nyoyPtJ0HAPACLpfRuX266tw+XbW5qFJzluTppWUFemlZgSb1i9f0saka1KOT7UwA7YBRCqDF6hqb\n9eX2/Zq9OF8b91SoU1iQbslK17WjeiouMtR2HgDAS/VPjNJjUwbrrkmZenlZgd5YWaiPN+3T8OTO\nuvGsVE3IjOWpIIAPY5QC+F57DtZqwY5Szd9RpmW55aprdCu5S5h+f1FfXTY0UWHBfBsBALSOhE4d\ndM8FZ+jWc3rpb6v36IUl+Zrx2lqFBQdoTHqMsjJilZURp+6dOthOBdCK+G0SwL9paHJrTcFBzT86\nRHNKqyVJPbuEacrwJJ2dGacx6TEKcPF8UQBA2+gYEqgbz0rRdaN6avGucn2VXaqvskv1xbb9kqSM\nrhGakBmnCRmxGtKzM3fvBbwcoxSASirrjp6GlmrJrnLVNDQrOMClkanRumpEkiZkxiklJtx2JgDA\nzwQGuL4en5lxus9xlFtWrfnZZZq/o1RzFufpmYW5iggN1LhescrKiNX4jFjFRfB0EsDbMEoBP9TU\n7Nb6PRWaf/Qvz9klVZKkhKhQXTy4uyZkxGlUWheFh/AtAgDgGYwxSo+LUHpchKaPS1VVXaOW5hz4\n5o+qH2/eJ0nq1z1SEzLilJURp0E9OvHIHsAL8Bsn4AdqG5q0o6RK2SVVWppTrkU7y3S4rkkBLqNh\nPTvr7smZOjszTr3iOvIyLgAArxARGqRJ/eI1qV+8HMfRtn2HtWBHmeZnl+rJ+Tl64qscdQ4L0rje\nsRqd1kVndItUr7gIdQjmhkmAp2GUAj6k2e1o94Ea7Sip0vaSKu0oOazskioVHqyV43x9TWxEiCb2\njdeEzDid1SuGFyoHAHg9Y4z6JkSpb0KUbpmQroraBi3aVa4F2aVasLNMH24oPnqdlNIlXBnxEcqM\nj1RGfITO6BahHp3D5OJEFbCGUQp4qQPV9d+cfmYfHZ8791eprtEtSXIZKTkmXH0TInXZkMSjP4D5\nwQsA8H2dwoJ14cAEXTgwQW63o8KDtcouOazt+6q+/sPtvsP6bGvJN3+wDQsOUO+uX/+czIyPUGa3\nSGXGR6hTWLDd/xDATzBKAQ/V1OxWeXWDSg7XqaSyTqVVddpzsPboCK1SWVX9N9d2CQ9WZrcIXT2y\n59d/9Y2PVK+uHRUaxEOUAAD+zeUySo4JV3JMuCb16/bN+2sbmvT/2rvbELmuMoDj/2dmdncSk01K\n3STatCra0oJiwKAVPxR8wVpF/aC0YkulbyKI4hdFEF+KoiL4gq+orVStVrG+hBItQhUtlNpoKxir\nMb4Ug23S1rZJ2uxmd+bxw9ydmZ3M7K44mzvu/H8wzD3nnnvnyXL27jznnHtz4PBx/tyVrN6+/yFu\nueef7TY7puvtQd2dZ2xg+3Sd7dN1dmyp8/RNU96vKg3JqpLSiLgY+BxQBb6emZ/o2T8FfBN4EfAo\ncGlm/mO4oUrrQ2ZydHaBw0Wyefho69VKPuc4cqxV/8jxOZq59NjJWoXztm/iovNmitHc1tKjmc1T\n5fxjJEn6P7Vxssaus7ey6+yt7brM5OFjc51bYB5sDQTf9ddHOdloLjm+WglmNk2xfXqqnai2k9bp\neqt+S53NUzWf1yCtYMWkNCKqwBeBVwGHgHsiYk9m/rGr2dXAY5n5vIi4DPgkcOlaBCyNgmYzefLk\nAsdmF1/zHJtd4Gjx3l3Xve/hY3McPjrHifnGKefcunGi+CNW5/wdm1vbW+rtuu3Tdc582qRLbyVJ\nWiMRwbbpOtum61x03ky7vtFMHn1yjsNPzLUGkY/OcqQYXH7o6CwPPPoUd//93zxxYv6Uc26crLJ9\nus7Mpik212vFa6LnvcZ0n7qnTdb8u6+xsJqZ0hcDBzPzbwARcQvwBqA7KX0D8OFi+wfAFyIiMrNn\nnkf672QmmdDIZKGRLDSbNJrJQjM77131843F+qXt5htNTi40mWu/GszNd20vNItyo9NmvrFk/4mT\njXZyeXxugZV6d60Sp/xxef5ZW3jlBa3R1G3FSOqO6TrbpqdcaitJ0oiqVoJtm+ts21znBWwZ2O7E\nycaSFVCHi1VQh4/N8sixOR58YpYDRzoD2I3eJVE9ImDTVCdh3TBZZapWYapWvE8slou6iUrP/tZ2\nvXifrFWoVYJqJdrvE9XKknKtUqFajSXtatXOcdUIInD2V0O1mqT0LOCfXeVDwEsGtcnMhYh4AjgT\neGQYQZ5ujz15kqtuuud/Ps9KScuKGfuAE/SrHfRZWbTu3r+4ne3yqQd32nSOz6JtFgd3l1v7u9ou\nHl/sb2bSzFa5mUW52Uo4m8u0OV3DGpO1/hfxxQv95vpEMcI5UYxm9h/l3FyfaO+rT1S8YEuSNEY2\nTFbb96+uJDM5Md9or6o62mel1fHZhSX1J4oB88dPzDM33+gacO8MtvcuM14rEVCJoFIkqJV2OZbs\na5W7tyHoJLWtBBeCrnKxPwB6yotfrbrbnxpbsa8rVnq2g/7f0fqeb7kfwjJW+hY4jK+J3732wnUx\nsbGapLTfj6s3VVhNGyLiOuA6gHPOOWcVH12OxVGp4ZxrbTprv+pBn9XvF5KeX+TuXYN+2dsXjD4X\nhyUXj+KEQefCU62cekFa6aLVuci1tmvVChPV7tG9rtG+ajGyt7ivGOGrVSrUimMGJZ2T1YpLY6Rl\n7Nq1q+wQJGndiQg2TtbYOFlj+3R9aOdtNpOTjd4VYA1mi6S1ma0VZEtXnHVWmi00ulajNZssNBdX\nqyULjWbXxEHXREN7UmHpxEKzqy4zaTY7kxHtCY1cOlHS2dc7GZJdkx7Fe7/JF+ip6+zsnZjp1Xei\nZmDbATtWOG65zxpnsf6ooMsAAATsSURBVNIPJCJeCnw4M19dlN8PkJkf72pze9HmroioAQ8BM8st\n3929e3fu27dvCP8ESZIkSdKoiYjfZubuldpVVnGue4BzI+I5ETEJXAbs6WmzB7iy2H4TcIf3k0qS\nJEmSVrLiGtXiHtF3ArfT+i9hbszM/RFxPbAvM/cANwDfioiDwL9pJa6SJEmSJC1rVTdOZuZeYG9P\n3Qe7tmeBNw83NEmSJEnSerea5buSJEmSJK0Jk1JJkiRJUmlMSiVJkiRJpTEplSRJkiSVxqRUkiRJ\nklQak1JJkiRJUmlMSiVJkiRJpTEplSRJkiSVxqRUkiRJklQak1JJkiRJUmkiM8v54IiHgQdK+XCd\nTk8HHik7CI09+6FGgf1Qo8K+qFFgPxwPz8rMmZUalZaUajxExL7M3F12HBpv9kONAvuhRoV9UaPA\nfqhuLt+VJEmSJJXGpFSSJEmSVBqTUq21r5YdgIT9UKPBfqhRYV/UKLAfqs17SiVJkiRJpXGmVJIk\nSZJUGpNSDUVE1CPiNxHx+4jYHxEfKeojIj4WEQci4v6IeFfZsWp9W6YvviIifhcR90XEnRHxvLJj\n1foXEdWIuDcibivKz4mIuyPiLxHxvYiYLDtGrX99+uHNEfHniPhDRNwYERNlx6j1r7cfdtV/PiKO\nlxWXRoNJqYZlDnh5Zr4Q2AVcHBEXAm8DzgbOz8wLgFvKC1FjYlBf/DLw1szcBXwH+ECJMWp8vBu4\nv6v8SeAzmXku8BhwdSlRadz09sObgfOBFwAbgGvKCEpjp7cfEhG7ga3lhKNRYlKqociWxVGuieKV\nwDuA6zOzWbQ7UlKIGhPL9MUEpov6LcC/SghPYyQidgKvBb5elAN4OfCDoslNwBvLiU7jorcfAmTm\n3uJamcBvgJ1lxafx0K8fRkQV+BTw3rLi0ugwKdXQFMsy7gOOAD/PzLuB5wKXRsS+iPhpRJxbbpQa\nBwP64jXA3og4BFwBfKLMGDUWPkvry1azKJ8JPJ6ZC0X5EHBWGYFprPT2w7Zi2e4VwM9Od1AaO/36\n4TuBPZn5YDkhaZSYlGpoMrNRLI3cCbw4Ip4PTAGzmbkb+BpwY5kxajwM6IvvAS7JzJ3AN4BPlxmj\n1reIeB1wJDN/213dp6mPwNeaGdAPu30J+FVm/vo0hqUx068fRsQzgTcDny8tMI2UWtkBaP3JzMcj\n4pfAxbRmAm4tdv2IVjIgnRZdffE1wAuLGVOA7+HMgNbWy4DXR8QlQJ3W0vHPAlsjolbMlu7EZeRa\nW6f0w4j4dmZeHhEfAmaAt5caocZBv+vhflrPgDjYurOBjRFxMDN9COGYcqZUQxERMxGxtdjeALwS\n+BPwY1r3UAFcBBwoJ0KNiwF98X5gS0ScVzR7FT0PW5CGKTPfn5k7M/PZwGXAHZn5VuAXwJuKZlcC\nPykpRI2BAf3w8oi4Bng18JbFZz5Ia2VAPzwjM3dk5rOL+qdMSMebM6UalmcANxU3rVeA72fmbRFx\nJ3BzRLwHOI5P+NPaG9QXrwVujYgmraeeXlVmkBpb7wNuiYiPAvcCN5Qcj8bTV4AHgLuKWaofZub1\n5YYkaZxF68FrkiRJkiSdfi7flSRJkiSVxqRUkiRJklQak1JJkiRJUmlMSiVJkiRJpTEplSRJkiSV\nxqRUkiRJklQak1JJkiRJUmlMSiVJkiRJpfkP6qSX1jV+pQ0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fab79980470>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = 1\n",
    "b = 40\n",
    "c = 1\n",
    "x = np.linspace(35, 45)\n",
    "\n",
    "gaussian = a * np.exp(-(x - b)**2 / (2*c**2))\n",
    "\n",
    "plt.figure(figsize=(16, 6))\n",
    "plt.plot(x, gaussian)\n",
    "plt.plot([39, 39], [0, 1], color=\"black\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "latex_envs": {
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 0
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
