{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pickle\n",
    "import tensorflow as tf\n",
    "import sys\n",
    "\n",
    "if sys.platform == \"win32\":\n",
    "    sys.path.append(r\"C:\\Users\\vik\\Dropbox\\Code\\Python\\structural_engineering\")\n",
    "else:\n",
    "    sys.path.append(\"/home/ritchie46/Dropbox/Code/Python/structural_engineering\")\n",
    "\n",
    "from anastruct.fem.system import SystemElements\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0., -1.,\n",
      "        1.,  0.,  0.]), -0.1, False)\n",
      "[[ 0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.]\n",
      " [ 1.  2.  0.  0.]]\n",
      "(array([ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  1.,  0.,  0., -1.,\n",
      "       -1.,  0.,  0.]), -0.1, False)\n",
      "[[ 0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.]\n",
      " [ 0.  3.  0.  0.]\n",
      " [ 1.  2.  0.  0.]]\n",
      "(array([ 0.,  0.,  0.,  0.,  0.,  0.,  1.,  0.,  0., -1.,  0.,  0., -1.,\n",
      "       -1.,  0.,  0.]), -0.1, False)\n",
      "[[ 0.  0.  0.  0.]\n",
      " [ 0.  0.  4.  0.]\n",
      " [ 0.  3.  0.  0.]\n",
      " [ 1.  2.  0.  0.]]\n",
      "(array([ 0.,  0.,  0.,  0.,  0.,  0., -1.,  0.,  0., -1.,  0.,  1., -1.,\n",
      "       -1.,  0.,  0.]), -0.1, False)\n",
      "[[ 0.  0.  0.  0.]\n",
      " [ 0.  0.  4.  0.]\n",
      " [ 0.  3.  0.  5.]\n",
      " [ 1.  2.  0.  0.]]\n",
      "(array([ 0.,  0.,  0.,  0.,  0.,  0., -1.,  0.,  0., -1.,  0., -1., -1.,\n",
      "       -1.,  0.,  1.]), 3.4441558772842846, True)\n",
      "[[ 0.  0.  0.  0.]\n",
      " [ 0.  0.  4.  0.]\n",
      " [ 0.  3.  0.  5.]\n",
      " [ 1.  2.  0.  6.]]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "class Environment:\n",
    "    def __init__(self, length=3, height=2, optimize='moment'):\n",
    "        self.length = length\n",
    "        self.height = height\n",
    "        self.state = None\n",
    "        self.n = None\n",
    "        self.actions_chosen = None\n",
    "        self.no_action = None\n",
    "        self.action_space = {0, 1, 2, 3, 4, 5, 6, 7}\n",
    "        self.valid_actions = None\n",
    "        self.optimize = optimize\n",
    "        self.result_map = {}\n",
    "        self.ss = None\n",
    "        self.current_distance = length - 1\n",
    "\n",
    "        # actions\n",
    "        right = 0\n",
    "        left = 4\n",
    "        up = 2\n",
    "        down = 6\n",
    "        up_right = 1\n",
    "        up_left = 3\n",
    "        down_right = 7\n",
    "        down_left = 5\n",
    "        \n",
    "        # If the state is a flattened array. This maps to the index displacements.\n",
    "        self.move_map = {right: 1,\n",
    "                         left: -1,\n",
    "                         up: -length,\n",
    "                         down: length,\n",
    "                         up_right: -length + 1,\n",
    "                         up_left: -length - 1,\n",
    "                         down_right: length + 1,\n",
    "                         down_left: length -1}\n",
    "            \n",
    "    def reset(self):\n",
    "        self.state = np.zeros((self.height, self.length))\n",
    "        self.n = 1\n",
    "        self.actions_chosen = 0\n",
    "        self.state[-1][0] = self.n\n",
    "        self.det_valid_actions()\n",
    "        return self.state.ravel()\n",
    "    \n",
    "    def return_action(self, r):\n",
    "        done = False\n",
    "        self.det_valid_actions()\n",
    "        s = np.array(self.state.ravel())\n",
    "        i = np.where(s == self.n)\n",
    "        s[s > 0] = -1\n",
    "        s[i] = 1\n",
    "                \n",
    "        # Bridge is build\n",
    "        if self.state[-1][-1] != 0:\n",
    "            r = r + 2 + 1 / (self.structure() / (1 / 16 * self.length**2))**2 # that is moment to the power 2\n",
    "            done = True\n",
    "            return s, r, done\n",
    "        \n",
    "        if len(self.valid_actions) == 0:\n",
    "            done = True\n",
    "            r -= 2\n",
    "        \n",
    "        return s, r, done\n",
    "    \n",
    "    def det_valid_actions(self):\n",
    "        no_action = set()\n",
    "        right = 0\n",
    "        left = 4\n",
    "        top = 2\n",
    "        down = 6\n",
    "        top_right = 1\n",
    "        top_left = 3\n",
    "        down_right = 7\n",
    "        down_left = 5\n",
    "        \n",
    "        # current location\n",
    "        row, col = np.where(self.state == self.n)\n",
    "                \n",
    "        # right:\n",
    "        try:\n",
    "            if self.state[row, col + 1] != 0:\n",
    "                no_action.add(right)\n",
    "        except IndexError:\n",
    "            no_action.add(right)\n",
    " \n",
    "        if col - 1 < 0:\n",
    "            no_action.add(left)\n",
    "        elif self.state[row, col - 1] != 0:\n",
    "            no_action.add(left)\n",
    "\n",
    "        if row - 1 < 0:\n",
    "            no_action.add(top)\n",
    "        elif self.state[row - 1, col] != 0:\n",
    "            no_action.add(top)\n",
    "            \n",
    "        try:\n",
    "            if self.state[row + 1, col] != 0:\n",
    "                no_action.add(down)\n",
    "        except IndexError:\n",
    "            no_action.add(down)\n",
    "            \n",
    "        if col -1 < 0 or row + 1 == self.height:\n",
    "            no_action.add(down_left)\n",
    "        elif self.state[row + 1, col - 1] != 0:\n",
    "            no_action.add(down_left)\n",
    "\n",
    "        try:\n",
    "            if self.state[row + 1, col + 1] != 0:\n",
    "                no_action.add(down_right)\n",
    "        except IndexError:\n",
    "            no_action.add(down_right)\n",
    "            \n",
    "        if row - 1 < 0 or col - 1 < 0:\n",
    "            no_action.add(top_left)\n",
    "        elif self.state[row -1, col - 1] != 0:\n",
    "            no_action.add(top_left)\n",
    "\n",
    "        if row - 1 < 0 or col + 1 == self.length:\n",
    "            no_action.add(top_right)\n",
    "        elif self.state[row - 1, col + 1] != 0:\n",
    "                no_action.add(top_right)\n",
    "            \n",
    "        self.no_action = no_action\n",
    "        self.valid_actions = list(self.action_space - no_action)\n",
    "        \n",
    "    \n",
    "    def step(self, a):\n",
    "        \"\"\"\n",
    "        :param a: (int) action direction\n",
    "        \n",
    "        → 0\n",
    "        ↗ 1\n",
    "        ↑ 2\n",
    "        ↖ 3\n",
    "        ← 4\n",
    "        ↙ 5\n",
    "        ↓ 6\n",
    "        ↘ 7\n",
    "        \"\"\"\n",
    "        self.actions_chosen += 1\n",
    "\n",
    "        flat_location_index = np.argwhere(self.state.ravel() == self.n)\n",
    "                                \n",
    "        if a in self.no_action:\n",
    "            return self.return_action(-0.2)\n",
    "        \n",
    "        # there is a valid action, make a move\n",
    "        self.n += 1\n",
    "        move = self.move_map[a]\n",
    "        self.state.ravel()[flat_location_index + move] = self.n\n",
    "        \n",
    "        row, col = np.where(self.state == self.n)\n",
    "        y = self.height - 1 - row[0]\n",
    "        x = self.length - 1  - col[0]\n",
    "        \n",
    "#         distance = (x**2 + y**2)**0.5\n",
    "#         d_distance = self.current_distance - distance\n",
    "#         self.current_distance = distance\n",
    "\n",
    "        return self.return_action(-0.1)\n",
    "    \n",
    "    def structure(self):\n",
    "        self.ss = SystemElements()\n",
    "        last_loc = [0, 0]\n",
    "        for i in range(2, self.n + 1):\n",
    "            row, col = np.where(i  == self.state)\n",
    "            \n",
    "            y = self.height - 1 - row[0]\n",
    "            x = col[0] \n",
    "\n",
    "            current_loc = [x, y]\n",
    "            self.ss.add_element([last_loc, [x, y]], g=-1)\n",
    "            last_loc = current_loc\n",
    "        \n",
    "#         n_nodes = len(self.ss.node_map)\n",
    "#         forces = -1 / (n_nodes - 2)\n",
    "#         for i in range(2, n_nodes):\n",
    "#             self.ss.point_load(node_id=i, Fz=forces)\n",
    "  \n",
    "        self.ss.add_support_hinged(1)\n",
    "        self.ss.add_support_hinged(len(self.ss.node_map))\n",
    "        self.ss.solve()\n",
    "        \n",
    "        f_max = np.max(np.abs(self.ss.get_element_result_range(self.optimize)))\n",
    "        \n",
    "        return f_max\n",
    "    \n",
    "    def sample_action(self):\n",
    "        return np.random.randint(0, 8) #self.valid_actions[np.random.randint(0, len(self.valid_actions))]\n",
    "\n",
    "    \n",
    "\n",
    "        \n",
    "def test_env():\n",
    "    env = Environment(4, 4)\n",
    "    s = env.reset()\n",
    "\n",
    "    print(env.step(0))\n",
    "    print(env.state)\n",
    "    print(env.step(2))\n",
    "    print(env.state)\n",
    "    print(env.step(1))\n",
    "    print(env.state)\n",
    "    print(env.step(7))\n",
    "    print(env.state)\n",
    "    print(env.step(6))\n",
    "    print(env.state)\n",
    "\n",
    "#     env.step(0)\n",
    "#     print(env.state, \"\\n\")\n",
    "#     env.step(7)\n",
    "#     print(env.state)\n",
    "#     env.structure()\n",
    "    \n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# https://theneuralperspective.com/2016/11/25/reinforcement-learning-rl-policy-gradients-i/\n",
    "\n",
    "def leaky_relu(x, alpha=0.1):\n",
    "    return tf.maximum(x, alpha * x)\n",
    "\n",
    "class Agent:\n",
    "    def __init__(self, data_size, hidden_size, action_space, learning_rate, name):\n",
    "        \"\"\"\n",
    "        :param data_size: (int) Columns of the data vector.\n",
    "        :param hidden_size: (int) No. of hidden nodes.\n",
    "        :param action_space: (int) No. of outputs.\n",
    "        :param learning_rate: (flt)\n",
    "        \"\"\"\n",
    "      \n",
    "        if len(hidden_size) == 1:\n",
    "            one_hidden_layer = True\n",
    "            hidden_size.append(hidden_size[0])\n",
    "        else:\n",
    "            one_hidden_layer = False\n",
    "\n",
    "        \n",
    "        # Step 1: Feed forward\n",
    "        # The argmax is the maximum Q-value.\n",
    "        self.input_s = tf.placeholder(tf.float32, [None, data_size], name=\"input_s\")\n",
    "        self.w1 = tf.get_variable(f\"{name}_w1\", shape=[data_size, hidden_size[0]], initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.b1 = tf.get_variable(f\"{name}_b1\", shape=(hidden_size[0], ), initializer=tf.zeros_initializer())\n",
    "        self.layer_1 = leaky_relu(tf.matmul(self.input_s, self.w1) + self.b1)\n",
    "        \n",
    "        self.w2 = tf.get_variable(f\"{name}_w2\", shape=[hidden_size[0], hidden_size[1]], initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.b2 = tf.get_variable(f\"{name}_b2\", shape=(hidden_size[1], ), initializer=tf.zeros_initializer())\n",
    "        self.layer_2 = leaky_relu(tf.matmul(self.layer_1, self.w2) + self.b2)\n",
    "                     \n",
    "        self.w_out = tf.get_variable(f\"{name}_w_out\", shape=[hidden_size[1], action_space], initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.b_out = tf.get_variable(f\"{name}_b_out\", shape=(action_space, ), initializer=tf.zeros_initializer())\n",
    "        \n",
    "        if one_hidden_layer:\n",
    "            hidden_layer = self.layer_1\n",
    "        else:\n",
    "            hidden_layer = self.layer_2\n",
    "        \n",
    "        # argmax(Q(s, a)) \n",
    "        self.predict_Q = tf.matmul(hidden_layer, self.w_out) + self.b_out # actual Q-value\n",
    "        self.p = tf.nn.softmax(self.predict_Q)\n",
    "        self.Q_a = tf.argmax(self.predict_Q, 1)\n",
    "        self.saver = tf.train.Saver()\n",
    "        \n",
    "        # Step 2: Determine loss / gradients. \n",
    "        # One hot encoded actions\n",
    "        self.executed_actions = tf.placeholder(tf.int32)\n",
    "        \n",
    "        self.one_hot = tf.one_hot(self.executed_actions, 8)\n",
    "        self.Q = tf.reduce_sum(tf.multiply(self.predict_Q, self.one_hot), axis=1)\n",
    "        self.next_Q_r = tf.placeholder(tf.float32)\n",
    "\n",
    "         # Loss\n",
    "         # mse: (     target      -    prediction)^2\n",
    "         #      r + max(Q(s', a') -    Q(s, a) )^2\n",
    "        \n",
    "        #self.loss = tf.losses.huber_loss(self.next_Q_r, - self.Q, delta=15)\n",
    "#         self.clipped_error = tf.maximum(tf.abs(self.next_Q_r - self.Q), tf.ones(tf.shape(self.Q)))\n",
    "#         self.loss = tf.reduce_sum(tf.square(self.clipped_error))       \n",
    "\n",
    "        starter_learning_rate = learning_rate\n",
    "        self.train_count = tf.Variable(0, trainable=False, name=f\"{name}_train_count\")\n",
    "        learning_rate = tf.train.exponential_decay(starter_learning_rate, self.train_count,\n",
    "                                                   1000, 0.96)\n",
    "    \n",
    "        self.loss = tf.reduce_sum(tf.square(self.next_Q_r - self.Q))   \n",
    "        optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "        \n",
    "        self.train = optimizer.minimize(self.loss, self.train_count)\n",
    "        \n",
    "        \n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def discounted_reward(r, gamma):\n",
    "    \"\"\"\n",
    "    The reward for a given state. Is the reward for that state + the discounted sum of future rewards.\n",
    "    \n",
    "    :param r: (array) Rewards.\n",
    "    :param gamma: (flt) Discount factor\n",
    "    \"\"\"\n",
    "    return np.cumsum(r * gamma**(np.arange(len(r)))[::-1])[::-1]\n",
    "\n",
    "def prepare_update_target(trainables, tau=0.05):\n",
    "    \"\"\"\n",
    "    The weights and biases of the target will be a depended of the primary network.\n",
    "    \n",
    "    wb[target] = tau * wb[primary] + (1-tau) * wb[target]\n",
    "    \n",
    "    This is a tensorflow operation and still needs to be run with Session.run(operation_holder)\n",
    "    \n",
    "    :trainables (tf.trainable_variables)\n",
    "    :tau (flt) Rate to update the target graph\n",
    "    \"\"\"\n",
    "    \n",
    "    operation_holder = []\n",
    "    n_variables = len(trainables) // 2 # the agent has half and the target has half\n",
    "    \n",
    "    for i, v in enumerate(trainables[0: n_variables]):\n",
    "        operation_holder.append(trainables[i + n_variables].assign(v.value() * tau) +\n",
    "                                ((1 - tau) * trainables[i + n_variables].value()))\n",
    "    return operation_holder\n",
    "\n",
    "def update_target(operation_holder, session):\n",
    "    for op in operation_holder:\n",
    "        session.run(op)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.667360347494 train_count 15488 loss 0.477816 eps 0.09990628614486748\n",
      "0.443707837601 train_count 15587 loss 1.38988 eps 0.09990628614486748\n",
      "0.524500981561 train_count 15686 loss 0.593845 eps 0.09990628614486748\n",
      "0.359304577248 train_count 15785 loss 4.05721 eps 0.09990628614486748\n",
      "0.374203011064 train_count 15884 loss 0.752228 eps 0.09990628614486748\n",
      "0.580970897654 train_count 15984 loss 0.668351 eps 0.09990628614486748\n",
      "0.876998766431 train_count 16084 loss 5.69235 eps 0.09990628614486748\n",
      "0.0465974575166 train_count 16178 loss 0.603223 eps 0.09990628614486748\n",
      "0.0747488582335 train_count 16276 loss 0.859723 eps 0.09990628614486748\n",
      "0.0416824847548 train_count 16373 loss 1.52317 eps 0.09990628614486748\n",
      "0.215243585896 train_count 16471 loss 4.26813 eps 0.09990628614486748\n",
      "0.539468365938 train_count 16570 loss 2.51492 eps 0.09990628614486748\n",
      "0.0675016616232 train_count 16667 loss 0.938396 eps 0.09990628614486748\n",
      "0.211260882325 train_count 16767 loss 1.46387 eps 0.09990628614486748\n",
      "0.101623406461 train_count 16865 loss 0.67447 eps 0.09990628614486748\n",
      "-0.123378754201 train_count 16960 loss 3.43497 eps 0.09990628614486748\n",
      "0.0591191103686 train_count 17057 loss 55.103 eps 0.09990628614486748\n",
      "0.0143498359155 train_count 17155 loss 2.58285 eps 0.09990628614486748\n",
      "0.0932387574004 train_count 17253 loss 2.82041 eps 0.09990628614486748\n",
      "0.218451295979 train_count 17351 loss 3.64749 eps 0.09990628614486748\n",
      "0.384775273901 train_count 17451 loss 4.34415 eps 0.09990628614486748\n",
      "0.20984679961 train_count 17548 loss 1.36758 eps 0.09990628614486748\n",
      "0.444519653928 train_count 17646 loss 3.15587 eps 0.09990628614486748\n",
      "-0.0906771169707 train_count 17740 loss 3.77309 eps 0.09990628614486748\n",
      "-0.0520069782439 train_count 17836 loss 2.13154 eps 0.09990628614486748\n",
      "0.349923459072 train_count 17936 loss 1.03072 eps 0.09990628614486748\n",
      "0.327600297472 train_count 18035 loss 1.62981 eps 0.09990628614486748\n",
      "0.762544380003 train_count 18133 loss 2.19067 eps 0.09990628614486748\n",
      "1.13968537324 train_count 18232 loss 5.1917 eps 0.09990628614486748\n",
      "0.965316180691 train_count 18332 loss 1.88149 eps 0.09990628614486748\n",
      "0.74757660094 train_count 18432 loss 2.14978 eps 0.09990628614486748\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-7-c69d7898a425>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     88\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     89\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>=\u001b[0m \u001b[0mbuffer_size\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 90\u001b[1;33m                 \u001b[0mbatch\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvstack\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     91\u001b[0m                 \u001b[0mbatch\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrandint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbuffer_size\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m500\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     92\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\lib\\site-packages\\numpy\\core\\shape_base.py\u001b[0m in \u001b[0;36mvstack\u001b[1;34m(tup)\u001b[0m\n\u001b[0;32m    228\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    229\u001b[0m     \"\"\"\n\u001b[1;32m--> 230\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0m_nx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0matleast_2d\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_m\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0m_m\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtup\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    231\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    232\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mhstack\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtup\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\lib\\site-packages\\numpy\\core\\shape_base.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    228\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    229\u001b[0m     \"\"\"\n\u001b[1;32m--> 230\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0m_nx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0matleast_2d\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_m\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0m_m\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtup\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    231\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    232\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mhstack\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtup\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\lib\\site-packages\\numpy\\core\\shape_base.py\u001b[0m in \u001b[0;36matleast_2d\u001b[1;34m(*arys)\u001b[0m\n\u001b[0;32m    105\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    106\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mary\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 107\u001b[1;33m         \u001b[0mres\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    108\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mres\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    109\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mres\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "\n",
    "from collections import deque\n",
    "# %matplotlib inline\n",
    "# %matplotlib notebook\n",
    "# # %load_ext autoreload\n",
    "# # %autoreload 2\n",
    "\n",
    "\n",
    "# fig = plt.figure(figsize=(12, 6))\n",
    "# ax = fig.add_subplot(111)\n",
    "# fig.show()\n",
    "# fig.canvas.draw()\n",
    "\n",
    "env = Environment(4, 4, \"moment\")\n",
    "env.reset()\n",
    "\n",
    "H = [32] # hidden neurons\n",
    "D = env.state.size # input (state of the environment)\n",
    "learning_rate = 1e-2\n",
    "gamma = 0.99 # discount factor\n",
    "epochs = 50000\n",
    "buffer_size = 5000\n",
    "max_frames = 150\n",
    "action_space = 8\n",
    "\n",
    "contin = 1\n",
    "\n",
    "if not contin:\n",
    "    eps = 0.3\n",
    "    tf.reset_default_graph()\n",
    "    agent = Agent(D, H, action_space, learning_rate, \"agent\")\n",
    "    target = Agent(D, H, action_space, learning_rate, \"target\")\n",
    "    \n",
    "    init = tf.global_variables_initializer()\n",
    "    sess = tf.Session()\n",
    "    sess.run(init)\n",
    "    buffer = deque()\n",
    "    \n",
    "    # the first half of the list are the variables of the agent.\n",
    "    # The last half the variables of the target\n",
    "    variables = tf.trainable_variables() \n",
    "    operation_holder = prepare_update_target(variables)\n",
    "    scores = deque()\n",
    "\n",
    "last_ep = 0\n",
    "\n",
    "\n",
    "#https://github.com/awjuliani/DeepRL-Agents/blob/master/Q-Network.ipynb\n",
    "train_count = 0\n",
    "loss = 0\n",
    "\n",
    "for ep in range(epochs):\n",
    "    \n",
    "    if eps > 0.1 and len(buffer) >= buffer_size:\n",
    "        eps *= 0.999\n",
    "    \n",
    "    if (ep + 1) % 100 == 0 and len(buffer) >= buffer_size:\n",
    "        print(np.mean(list(scores)[-1000:]), \"train_count\", train_count,\n",
    "              \"loss\", loss, \"eps\", eps)\n",
    "        \n",
    "    s = env.reset()\n",
    "    s = [s]\n",
    "    for c in range(max_frames):\n",
    "        \n",
    "        Q = sess.run(agent.predict_Q, {agent.input_s: s})\n",
    "        \n",
    "        if np.random.rand(1) < eps:\n",
    "            rand = True\n",
    "            a = env.sample_action()\n",
    "        else:\n",
    "            rand = False\n",
    "            a = np.argmax(Q)\n",
    "        \n",
    "        s_new, r, done = env.step(a)\n",
    "        scores.append(r)\n",
    "        \n",
    "        if rand == True:\n",
    "            buffer.append([s, a, r, s_new, done])\n",
    "        if rand == False and c % 50 == 0:\n",
    "            buffer.append([s, a, r, s_new, done])\n",
    "        \n",
    "        if len(buffer) > buffer_size:\n",
    "            buffer.popleft()\n",
    "            scores.popleft()\n",
    "        s = [s_new]\n",
    "                \n",
    "        if done:\n",
    "            \n",
    "            if len(buffer) >= buffer_size:\n",
    "                batch = np.vstack(buffer)\n",
    "                batch = batch[np.random.randint(0, buffer_size, size=500)]\n",
    "\n",
    "                s = np.vstack(batch[:, 0])\n",
    "                s_new = np.vstack(batch[:, 3])\n",
    "                r = batch[:, 2]\n",
    "                a = batch[:, 1]\n",
    "                done_ = np.array(batch[:, 4], dtype=bool)\n",
    "                Q = sess.run(agent.predict_Q, {agent.input_s: s})\n",
    "\n",
    "                # Double Q-learning primary network chooses an action:\n",
    "                Q_i = np.argmax(sess.run(agent.predict_Q, {agent.input_s: s_new}), 1)\n",
    "                \n",
    "                # Target network produces the Q-value of the chosen action\n",
    "                Q_new = sess.run(target.predict_Q, {target.input_s: s_new})\n",
    "                max_Q_new =Q_new[np.arange(Q_i.size)[:, None], Q_i[:, None]][:, -1]\n",
    "                \n",
    "                target_Q = r + gamma * max_Q_new\n",
    "                target_Q[done_] = r[done_]\n",
    "  \n",
    "                train_count, loss, _ = sess.run([agent.train_count, agent.loss, agent.train], \n",
    "                                                feed_dict={agent.input_s: s, \n",
    "                                                           agent.executed_actions: a, \n",
    "                                                           agent.next_Q_r: target_Q})\n",
    "                if c % 3 == 0:\n",
    "                    # update target network\n",
    "                    update_target(operation_holder, sess)\n",
    "\n",
    "            break\n",
    "            \n",
    "            \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 406,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32749"
      ]
     },
     "execution_count": 406,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(buffer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/ritchie46/Downloads/model_anastruct/model_bridge_4_2_moment.ckpt'"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agent.saver.save(sess, \"/home/ritchie46/Downloads/model_anastruct/model_bridge_4_2_moment.ckpt\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'G:\\\\bridge_builder\\\\model_bridge_4_2_axial\\\\model.ckpt'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agent.saver.save(sess, r\"G:\\bridge_builder\\model_bridge_4_2_axial\\model.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " [[ 0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.]\n",
      " [ 1.  0.  0.  0.]]\n",
      "[[-0.12808736 -0.09334762 -0.10103387 -0.23614739 -0.23016927 -0.19282101\n",
      "  -0.19304731 -0.2984907 ]]\n",
      "action 1\n",
      "-0.1 False\n",
      "\n",
      " [[ 0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.]\n",
      " [ 0.  2.  0.  0.]\n",
      " [ 1.  0.  0.  0.]]\n",
      "[[-0.10368284 -0.11574875 -0.12774605 -0.12350658 -0.1642811  -0.20029946\n",
      "  -0.16407546 -0.08707853]]\n",
      "action 7\n",
      "-0.1 False\n",
      "\n",
      " [[ 0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.]\n",
      " [ 0.  2.  0.  0.]\n",
      " [ 1.  0.  3.  0.]]\n",
      "[[ 2.46880364 -0.09685077 -0.15725884 -0.14345071 -0.17737159 -0.20034733\n",
      "  -0.20910886 -0.16599104]]\n",
      "action 0\n",
      "2.51404717866 True\n",
      "\n",
      " [[ 0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.]\n",
      " [ 0.  2.  0.  0.]\n",
      " [ 1.  0.  3.  4.]]\n",
      "[[-0.11165825 -0.13312925 -0.15180516 -0.14280865 -0.20325702 -0.18633628\n",
      "  -0.0402295   0.09051565]]\n",
      "action 7\n",
      "2.41404717866 True\n",
      "\n",
      " [[ 0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.]\n",
      " [ 0.  2.  0.  0.]\n",
      " [ 1.  0.  3.  4.]]\n",
      "[[-0.11165825 -0.13312925 -0.15180516 -0.14280865 -0.20325702 -0.18633628\n",
      "  -0.0402295   0.09051565]]\n",
      "action 7\n",
      "2.41404717866 True\n",
      "\n",
      " [[ 0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.]\n",
      " [ 0.  2.  0.  0.]\n",
      " [ 1.  0.  3.  4.]]\n",
      "[[-0.11165825 -0.13312925 -0.15180516 -0.14280865 -0.20325702 -0.18633628\n",
      "  -0.0402295   0.09051565]]\n",
      "action 7\n",
      "2.41404717866 True\n",
      "\n",
      " [[ 0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.]\n",
      " [ 0.  2.  0.  0.]\n",
      " [ 1.  0.  3.  4.]]\n",
      "[[-0.11165825 -0.13312925 -0.15180516 -0.14280865 -0.20325702 -0.18633628\n",
      "  -0.0402295   0.09051565]]\n",
      "action 7\n",
      "2.41404717866 True\n",
      "\r",
      " 11.9702358933 1.27614237492\n"
     ]
    }
   ],
   "source": [
    "env = Environment(4, 4)\n",
    "s = env.reset()\n",
    "\n",
    "\n",
    "\"\"\"    \n",
    "    → 0\n",
    "    ↗ 1\n",
    "    ↑ 2\n",
    "    ↖ 3\n",
    "    ← 4\n",
    "    ↙ 5\n",
    "    ↓ 6\n",
    "    ↘ 7\n",
    "\"\"\"\n",
    "\n",
    "bot = agent\n",
    "\n",
    "total_r = 0\n",
    "j = 0\n",
    "for a in [1, 0, 7, 7, 1, 0, 1]:\n",
    "    j += 1\n",
    "\n",
    "    print(\"\\n\", env.state)\n",
    "    \n",
    "    \n",
    "    a_dst = sess.run(bot.predict_Q, {bot.input_s: [s]})\n",
    "    a = np.argmax(a_dst)\n",
    "    #a = np.random.choice(np.arange(8), p=a_dst[0])\n",
    "\n",
    "    s, r, d = env.step(a)\n",
    "\n",
    "    print(a_dst)\n",
    "    print(\"action\", a)\n",
    "    print(r, d)\n",
    "    total_r += r\n",
    "    \n",
    "#     if d == True:\n",
    "#         j = 0\n",
    "#         print(env.state)\n",
    "#         break\n",
    "#         #env.reset()\n",
    "    \n",
    "print(\"\\r\", total_r, env.structure())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0., -1., -1.,  0., -1.,\n",
       "         0.,  0.,  1.]), 10.140799116945235, True)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env = Environment(4, 4)\n",
    "env.reset()\n",
    "env.step(1)\n",
    "env.step(0)\n",
    "env.step(7)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  },
  "latex_envs": {
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 0
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
