{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "import unicodedata\n",
    "import numpy as np\n",
    "import torch\n",
    "from torch import nn\n",
    "import re\n",
    "import os\n",
    "from tensorboardX import SummaryWriter\n",
    "\n",
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# download the needed data\n",
    "if not os.path.isfile('data.zip'):\n",
    "    ! curl -o data.zip https://download.pytorch.org/tutorial/data.zip && unzip data.zip "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " de question !\n",
      "Really?\tVraiment ?\n",
      "Really?\tVrai ?\n",
      "Really?\tAh bon ?\n",
      "Thanks.\tMerci !\n",
      "We try.\tOn essaye.\n",
      "We won.\tNous avons gagné.\n",
      "We won.\tNous gagnâmes.\n",
      "We won.\tNous l'avons emporté.\n",
      "We won.\tNous l'empor\n"
     ]
    }
   ],
   "source": [
    "# Take a quick view of the data.\n",
    "with open('data/eng-fra.txt') as f:\n",
    "    f.seek(1000)\n",
    "    print(f.read(200))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class Lang:\n",
    "    \"\"\"\n",
    "    Utility class that serves as a language dictionary\n",
    "    \"\"\"\n",
    "    def __init__(self, name):\n",
    "        self.name = name\n",
    "        # Count how often a word occurs in the language data.\n",
    "        self.word2count = {}\n",
    "        # Words are mapped to indices and vice versa\n",
    "        self.index2word = {0: \"SOS\", 1: \"EOS\"}\n",
    "        self.word2index = {v:k for k, v in self.index2word.items()}\n",
    "        # Total word count\n",
    "        self.n_words = 2  # Count SOS and EOS\n",
    "\n",
    "    def add_sentence(self, sentence):\n",
    "        \"\"\"\n",
    "        Process words in a sentence string.\n",
    "        \n",
    "        :param sentence: (str) \n",
    "        \"\"\"\n",
    "        for word in sentence.split(' '):\n",
    "            self.add_word(word)\n",
    "\n",
    "    def add_word(self, word):\n",
    "        \"\"\"\n",
    "        Process words\n",
    "        :param word: (str)\n",
    "        \"\"\"\n",
    "        if word not in self.word2index:\n",
    "            self.word2index[word] = self.n_words\n",
    "            self.word2count[word] = 1\n",
    "            self.index2word[self.n_words] = word\n",
    "            self.n_words += 1\n",
    "        elif word != 'SOS' and word != 'EOS':\n",
    "            self.word2count[word] += 1\n",
    "    \n",
    "    def translate_indexes(self, idx):\n",
    "        \"\"\"\n",
    "        Takes in a vector of indices and returns the sentence.\n",
    "        \"\"\"\n",
    "        return [self.index2word[i] for i in idx]\n",
    "    \n",
    "# Turn a Unicode string to plain ASCII, thanks to\n",
    "# http://stackoverflow.com/a/518232/2809427\n",
    "def unicode2ascii(s):\n",
    "    return ''.join(\n",
    "        c for c in unicodedata.normalize('NFD', s)\n",
    "        if unicodedata.category(c) != 'Mn'\n",
    "    )\n",
    "\n",
    "# Lowercase, trim, and remove non-letter characters\n",
    "def normalize_string(s):\n",
    "    s = unicode2ascii(s.lower().strip())\n",
    "    s = re.sub(r\"\\s?[.!?]\", r\" EOS\", s)\n",
    "    s = re.sub(r\"[^a-zA-Z.!?]+\", r\" \", s)\n",
    "    return s\n",
    "\n",
    "def read_langs(lang1, lang2, reverse=False):\n",
    "    print(\"Reading lines...\")\n",
    "\n",
    "    # Read the file and split into lines\n",
    "    lines = open('data/%s-%s.txt' % (lang1, lang2), encoding='utf-8').\\\n",
    "        read().strip().split('\\n')\n",
    "    \n",
    "    # Split every line into pairs and normalize\n",
    "    pairs = [[normalize_string(s) for s in l.split('\\t')] for l in lines]\n",
    "    # Reverse pairs, make Lang instances\n",
    "    if reverse:\n",
    "        pairs = [list(reversed(p)) for p in pairs]\n",
    "        input_lang = Lang(lang2)\n",
    "        output_lang = Lang(lang1)\n",
    "    else:\n",
    "        input_lang = Lang(lang1)\n",
    "        output_lang = Lang(lang2)\n",
    "\n",
    "    return input_lang, output_lang, pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Since there are a lot of example sentences and we want to train something quickly, we'll trim the data set to only relatively short and simple sentences. \n",
    "# Here the maximum length is 10 words (that includes ending punctuation) and we're filtering to sentences that translate to the form \"I am\" or \"He is\" etc. \n",
    "# (accounting for apostrophes replaced earlier).\n",
    "\n",
    "\n",
    "\n",
    "def filter_pairs(pairs):\n",
    "    MAX_LENGTH = 10\n",
    "    \n",
    "    eng_prefixes = (\n",
    "    \"i am \", \"i m \",\n",
    "    \"he is\", \"he s \",\n",
    "    \"she is\", \"she s\",\n",
    "    \"you are\", \"you re \",\n",
    "    \"we are\", \"we re \",\n",
    "    \"they are\", \"they re \"\n",
    "    )\n",
    "    \n",
    "    def filter_pair(p):\n",
    "        return len(p[0].split(' ')) < MAX_LENGTH and \\\n",
    "            len(p[1].split(' ')) < MAX_LENGTH \\\n",
    "            and p[0].startswith(eng_prefixes)\n",
    "    return [pair for pair in pairs if filter_pair(pair)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Data:\n",
    "    def __init__(self, pairs, lang_1, lang_2):\n",
    "        self.pairs = np.array(pairs)\n",
    "        np.random.seed(9)\n",
    "        np.random.shuffle(self.pairs)\n",
    "        idx_1 = [[lang_1.word2index[word] for word in s.split(' ')] \n",
    "                               for s in self.pairs[:, 0]]\n",
    "        idx_2 = [[lang_2.word2index[word] for word in s.split(' ')]\n",
    "                               for s in self.pairs[:, 1]]\n",
    "        self.idx_pairs = np.array(list(zip(idx_1, idx_2)))\n",
    "        self.shuffle_idx = np.arange(len(pairs))\n",
    "                \n",
    "    def __str__(self):\n",
    "        return(self.pairs)\n",
    "    \n",
    "    def shuffle(self):\n",
    "        np.random.shuffle(self.shuffle_idx)\n",
    "        self.pairs = self.pairs[self.shuffle_idx]\n",
    "        self.idx_pairs = self.idx_pairs[self.shuffle_idx]      \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading lines...\n",
      "Read 135842 sentence pairs\n",
      "Trimmed to 10853 sentence pairs\n",
      "Counting words...\n",
      "Counted words:\n",
      "eng 2922\n",
      "fra 4486\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['we are even EOS', 'nous sommes a egalite EOS'], dtype='<U60')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def prepare_data(lang1, lang2, reverse=False):\n",
    "    # read_langs initialized the Lang objects (still empty) and returns the pair sentences.\n",
    "    input_lang, output_lang, pairs = read_langs(lang1, lang2, reverse)\n",
    "    print(\"Read %s sentence pairs\" % len(pairs))\n",
    "    \n",
    "    # Reduce data. We haven't got all day to train a model.\n",
    "    pairs = filter_pairs(pairs) \n",
    "    print(\"Trimmed to %s sentence pairs\" % len(pairs))\n",
    "    print(\"Counting words...\")\n",
    "    \n",
    "    # Process the language pairs.\n",
    "    for pair in pairs:\n",
    "        input_lang.add_sentence(pair[0])\n",
    "        output_lang.add_sentence(pair[1])\n",
    "    print(\"Counted words:\")\n",
    "    print(input_lang.name, input_lang.n_words)\n",
    "    print(output_lang.name, output_lang.n_words)\n",
    "    return input_lang, output_lang, Data(pairs, input_lang, output_lang)\n",
    "\n",
    "\n",
    "eng, fra, data = prepare_data('eng', 'fra', False)\n",
    "data.pairs[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Encoder\n",
    "\n",
    "The encoder of a seq2seq network is a RNN that outputs some value for every word from the input sentence. For every input word the encoder outputs a vector and a hidden state, and uses the hidden state for the next input word.\n",
    "\n",
    "![](img/encoder-network.png)\n",
    "\n",
    "Every output could be seen as the context of the sentence up to that point.\n",
    "\n",
    "![](img/training_seq2seq_many2may.svg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([5, 1, 2])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(self, n_words, embedding_size, hidden_size, bidirectional=False, device=device):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.bidirectional = bidirectional\n",
    "        self.hidden_size = hidden_size\n",
    "        # The word embeddings will also be trained\n",
    "        # To freeze them --> m.embedding.weight.requires_grad = False\n",
    "        self.embedding = nn.Embedding(n_words, embedding_size)  \n",
    "        self.rnn = nn.GRU(embedding_size, hidden_size, bidirectional=bidirectional)\n",
    "        self.device = device\n",
    "        if device == 'cuda':\n",
    "            self.cuda()\n",
    "                    \n",
    "    def forward(self, x):\n",
    "        # shape (seq_length, batch_size, input_size)\n",
    "        dense_vector = self.embedding(x).view(x.shape[0], 1, -1)\n",
    "        \n",
    "        # init hidden layer at beginning of sequence\n",
    "        n = 2 if self.bidirectional else 1\n",
    "        \n",
    "        h = torch.zeros(n, 1, self.hidden_size, device=self.device)\n",
    "            \n",
    "        x, h = self.rnn(dense_vector, h)\n",
    "\n",
    "        return x, h\n",
    "        \n",
    "\n",
    "m = Encoder(eng.n_words, 10, 2, False, 'cpu')\n",
    "scentence = torch.tensor([400, 1, 2, 6, 8])\n",
    "a = m(scentence)\n",
    "a[0].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple Decoder\n",
    "\n",
    "In the simplest seq2seq decoder we use only last output of the encoder. This last output is sometimes called the context vector as it encodes context from the entire sequence. This context vector is used as the initial hidden state of the decoder.\n",
    "\n",
    "At every step of decoding, the decoder is given an input token and hidden state. The initial input token is the start-of-string <SOS> token, and the first hidden state is the context vector (the encoder’s last hidden state).\n",
    "    \n",
    "![](img/decoder-network.png)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-23351.4941, grad_fn=<SumBackward0>)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(self, embedding_size, hidden_size, output_size, device=device):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.decoder = 'simple'\n",
    "        self.hidden_size = hidden_size\n",
    "        # Lookup table for the last word activation.\n",
    "        self.embedding = nn.Embedding(output_size, embedding_size)\n",
    "        self.relu = nn.LeakyReLU()\n",
    "        self.rnn = nn.GRU(embedding_size, hidden_size)\n",
    "        self.out = nn.Sequential(\n",
    "            nn.LeakyReLU(),\n",
    "            nn.Linear(hidden_size, output_size),\n",
    "            nn.LogSoftmax(2)\n",
    "        )\n",
    "        self.device = device\n",
    "        if device == 'cuda':\n",
    "            self.cuda()\n",
    "            \n",
    "    def forward(self, word, h):\n",
    "        \"\"\"\n",
    "        :param word: (tensor) Last word or start of sentence token.\n",
    "        :param h: (tensor) Hidden state or context tensor.\n",
    "        \"\"\"\n",
    "        # map from shape (seq_len, embedding_size) to (seq_len, batch, embedding_size) (Notel: seq length is the number of words in the sentence)\n",
    "        word_embedding = self.embedding(word).view(h.shape[0], 1, -1)\n",
    "        a = self.relu(word_embedding)\n",
    "        x, h = self.rnn(a, h)\n",
    "\n",
    "        return self.out(x), h\n",
    "\n",
    "m = Decoder(10, 20, eng.n_words, device='cpu')\n",
    "m.train(False)\n",
    "m(torch.tensor([1]) ,torch.zeros(1, 1, 20))[0].sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](img/attention-decoder-network.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([3, 1, 256])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 1, 2])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class AttentionDecoder(nn.Module):\n",
    "    def __init__(self, embedding_size, hidden_size, output_size, dropout=0.1, max_length=10, device=device):\n",
    "        super(AttentionDecoder, self).__init__()\n",
    "        self.decoder = 'attention'\n",
    "        self.max_length = max_length\n",
    "        self.device = device\n",
    "        self.embedding = nn.Sequential(\n",
    "            nn.Embedding(output_size, embedding_size),\n",
    "            nn.Dropout(dropout)\n",
    "        )\n",
    "        \n",
    "        # Seperate neural network to learn the attention weights\n",
    "        self.attention_weights = nn.Sequential(\n",
    "            nn.Linear(embedding_size + hidden_size, max_length),\n",
    "            nn.Softmax(2)\n",
    "        )\n",
    "        self.attention_combine = nn.Sequential(\n",
    "            nn.Linear(hidden_size + embedding_size, hidden_size),\n",
    "            nn.ReLU()\n",
    "        )\n",
    "\n",
    "        self.rnn = nn.GRU(hidden_size, hidden_size)\n",
    "        self.out = nn.Sequential(\n",
    "            nn.Linear(hidden_size, output_size),\n",
    "            nn.LogSoftmax(2)\n",
    "            )\n",
    "        \n",
    "        if device == 'cuda':\n",
    "            self.cuda()\n",
    "        \n",
    "    def forward(self, word, h, encoder_outputs):\n",
    "        \"\"\"\n",
    "        :param word: (LongTensor) The word indices. This is the last activated word or \n",
    "        :param h: (tensor) The hidden state from the previous step. In the first step, the hidden state of the encoder.\n",
    "        :param encoder_outputs: (tensor) Zero padded (max_length, shape, shape) outputs from the encoder.\n",
    "        \"\"\"\n",
    "        # map from shape (seq_len, embedding_size) to (seq_len, batch, embedding_size) (Notel: seq length is the number of words in the sentence)\n",
    "        word_embedding = self.embedding(word).view(1, 1, -1)\n",
    "        \n",
    "        # Concatenate the word embedding and the last hidden state, so that attention weights can be determined.\n",
    "        x = torch.cat((word_embedding, h), 2)\n",
    "        attention_weights = self.attention_weights(x)\n",
    "        # attention applied\n",
    "        x = torch.bmm(attention_weights, encoder_outputs.unsqueeze(0))  # could also be done with matmul\n",
    "   \n",
    "        # attention combined\n",
    "        x = torch.cat((word_embedding, x), 2)\n",
    "        x = self.attention_combine(x)\n",
    "        \n",
    "        x, h = self.rnn(x, h)\n",
    "       \n",
    "        x = self.out(x)\n",
    "\n",
    "        return x, h\n",
    "\n",
    "\n",
    "embedding_size = 256\n",
    "hidden_size = 256\n",
    "max_length = 10\n",
    "\n",
    "m = Encoder(eng.n_words, embedding_size, hidden_size, bidirectional=False, device='cpu')\n",
    "scentence = torch.tensor([1, 23, 9])\n",
    "out, h = m(scentence)\n",
    "print(out.shape)\n",
    "\n",
    "encoder_outputs = torch.zeros(max_length, out.shape[-1], device='cpu')\n",
    "encoder_outputs[:out.shape[0], :out.shape[-1]] = out.view(out.shape[0], -1)\n",
    "\n",
    "\n",
    "m = AttentionDecoder(embedding_size, hidden_size, 2, device='cpu')\n",
    "m(torch.tensor([1]), h, encoder_outputs)[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_decoder(decoder, criterion, sentence, h, teacher_forcing=False, encoder_outputs=None):\n",
    "    loss = 0\n",
    "    word = torch.tensor([0], device=device) # <SOS>\n",
    "    for j in range(sentence.shape[0]):\n",
    "        if decoder.decoder == 'attention':\n",
    "            x, h = decoder(word, h, encoder_outputs)\n",
    "        else:\n",
    "            x, h = decoder(word, h)\n",
    "\n",
    "        loss += criterion(x.view(1, -1), sentence[j].view(-1))\n",
    "        if teacher_forcing:\n",
    "            word = sentence[j]\n",
    "        else:\n",
    "            word = x.argmax().detach()\n",
    "        if word.item() == 1: # <EOS>\n",
    "            break\n",
    "    return loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "teacher_forcing_ratio = 0.5\n",
    "\n",
    "embedding_size = 100\n",
    "context_vector_size = 256\n",
    "bidirectional = False\n",
    "encoder = Encoder(eng.n_words, embedding_size, context_vector_size, bidirectional)\n",
    "context_vector_size = context_vector_size * 2 if bidirectional else context_vector_size \n",
    "decoder = AttentionDecoder(embedding_size, context_vector_size, fra.n_words)\n",
    "writer = SummaryWriter('tb/train3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cuda'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decoder.device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 0\n",
      "epoch 1\n",
      "epoch 2\n",
      "epoch 3\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def train(encoder, decoder):\n",
    "    criterion = nn.NLLLoss()\n",
    "    optim_encoder = torch.optim.SGD(encoder.parameters(), lr=0.01)\n",
    "    optim_decoder = torch.optim.SGD(decoder.parameters(), lr=0.01)\n",
    "\n",
    "    epochs = 4\n",
    "    batch_size = 1\n",
    "\n",
    "    encoder.train(True)\n",
    "    decoder.train(True)\n",
    "\n",
    "    for epoch in range(epochs):\n",
    "        data.shuffle()\n",
    "\n",
    "        for i in range(data.pairs.shape[0]):\n",
    "            optim_decoder.zero_grad()\n",
    "            optim_encoder.zero_grad()\n",
    "            \n",
    "            pair = data.idx_pairs[i]\n",
    "\n",
    "            eng_sentence = torch.tensor(pair[0], device=device)\n",
    "            fra_sentence = torch.tensor(pair[1], device=device)\n",
    "\n",
    "            # Encode the input language\n",
    "            out, h = encoder(eng_sentence)        \n",
    "            encoder_outputs = torch.zeros(max_length, out.shape[-1], device=device)\n",
    "            \n",
    "            if decoder.decoder == 'attention':\n",
    "                encoder_outputs[:out.shape[0], :out.shape[-1]] = out.view(out.shape[0], -1)\n",
    "\n",
    "            teacher_forcing = np.random.rand() < teacher_forcing_ratio\n",
    "            loss = run_decoder(decoder, criterion, fra_sentence, h, teacher_forcing, encoder_outputs)\n",
    "\n",
    "            loss.backward()\n",
    "            writer.add_scalar('loss', loss.cpu().item() / (len(fra_sentence)))\n",
    "\n",
    "            optim_decoder.step()\n",
    "            optim_encoder.step()\n",
    "\n",
    "        print(f'epoch {epoch}')\n",
    "\n",
    "train(encoder, decoder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = AttentionDecoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "English scentence:\t we re all single\n",
      "French scentence:\t nous sommes toutes celibataires\n",
      "\n",
      "Model translation:\t nous sommes toutes celibataire \n",
      "\n",
      "\n",
      "English scentence:\t he is amusing himself by playing video games\n",
      "French scentence:\t il s amuse en jouant aux jeux videos\n",
      "\n",
      "Model translation:\t il est plein a des anglais \n",
      "\n",
      "\n",
      "English scentence:\t i m an artist\n",
      "French scentence:\t je suis une artiste\n",
      "\n",
      "Model translation:\t je suis un artiste \n",
      "\n",
      "\n",
      "English scentence:\t he s enjoying himself\n",
      "French scentence:\t il s amuse\n",
      "\n",
      "Model translation:\t il est est a \n",
      "\n",
      "\n",
      "English scentence:\t he is an american\n",
      "French scentence:\t il est americain\n",
      "\n",
      "Model translation:\t il est un \n",
      "\n",
      "\n",
      "English scentence:\t you re the oldest\n",
      "French scentence:\t tu es le plus vieux\n",
      "\n",
      "Model translation:\t vous etes le plus \n",
      "\n",
      "\n",
      "English scentence:\t he s about to leave\n",
      "French scentence:\t il est sur le point de partir\n",
      "\n",
      "Model translation:\t il est sur le point \n",
      "\n",
      "\n",
      "English scentence:\t i m really happy\n",
      "French scentence:\t je suis vraiment heureuse\n",
      "\n",
      "Model translation:\t je suis vraiment contente \n",
      "\n",
      "\n",
      "English scentence:\t we re all hungry\n",
      "French scentence:\t nous avons tous faim\n",
      "\n",
      "Model translation:\t nous avons toutes faim \n",
      "\n",
      "\n",
      "English scentence:\t i m not giving you any more money\n",
      "French scentence:\t je ne te donnerai pas davantage d argent\n",
      "\n",
      "Model translation:\t je ne te donnerai d argent d argent \n",
      "\n",
      "\n",
      "English scentence:\t he is independent of his parents\n",
      "French scentence:\t il est independant de ses parents\n",
      "\n",
      "Model translation:\t il est plein de ses parents \n",
      "\n",
      "\n",
      "English scentence:\t i am not a doctor but a teacher\n",
      "French scentence:\t je ne suis pas medecin mais professeur\n",
      "\n",
      "Model translation:\t je ne suis pas medecin mais enseignant \n",
      "\n",
      "\n",
      "English scentence:\t you are responsible for the result\n",
      "French scentence:\t vous etes responsable des resultats\n",
      "\n",
      "Model translation:\t vous es responsable de l \n",
      "\n",
      "\n",
      "English scentence:\t i m sympathetic\n",
      "French scentence:\t j eprouve de la compassion\n",
      "\n",
      "Model translation:\t je ne suis \n",
      "\n",
      "\n",
      "English scentence:\t you are drunk\n",
      "French scentence:\t tu es saoul\n",
      "\n",
      "Model translation:\t vous etes impressionnee \n",
      "\n",
      "\n",
      "English scentence:\t you aren t as short as i am\n",
      "French scentence:\t tu n es pas aussi petite que moi\n",
      "\n",
      "Model translation:\t vous n etes pas aussi petit que moi \n",
      "\n",
      "\n",
      "English scentence:\t i m resilient\n",
      "French scentence:\t je suis endurante\n",
      "\n",
      "Model translation:\t je suis endurant \n",
      "\n",
      "\n",
      "English scentence:\t you re wrong again\n",
      "French scentence:\t vous avez a nouveau tort\n",
      "\n",
      "Model translation:\t tu as tort \n",
      "\n",
      "\n",
      "English scentence:\t i am thinking of my vacation\n",
      "French scentence:\t je pense a mes vacances\n",
      "\n",
      "Model translation:\t je songe de mes enfants \n",
      "\n",
      "\n",
      "English scentence:\t she is able to sing very well\n",
      "French scentence:\t elle sait tres bien chanter\n",
      "\n",
      "Model translation:\t elle va bien bien bien bien \n",
      "\n",
      "\n",
      "English scentence:\t i m lazy\n",
      "French scentence:\t je suis faineant\n",
      "\n",
      "Model translation:\t je suis paresseux \n",
      "\n",
      "\n",
      "English scentence:\t he is a cruel person\n",
      "French scentence:\t c est un homme cruel\n",
      "\n",
      "Model translation:\t il est une personne \n",
      "\n",
      "\n",
      "English scentence:\t he is a poet\n",
      "French scentence:\t c est un poete\n",
      "\n",
      "Model translation:\t il est un \n",
      "\n",
      "\n",
      "English scentence:\t we re plastered\n",
      "French scentence:\t nous sommes bourres\n",
      "\n",
      "Model translation:\t nous sommes bourrees \n",
      "\n",
      "\n",
      "English scentence:\t she suddenly fell silent\n",
      "French scentence:\t elle se tut soudain\n",
      "\n",
      "Model translation:\t elle est deux deux \n",
      "\n",
      "\n",
      "English scentence:\t you re not making this easy\n",
      "French scentence:\t vous ne rendez pas ca facile\n",
      "\n",
      "Model translation:\t vous ne fais pas ca \n",
      "\n",
      "\n",
      "English scentence:\t she s very afraid of dogs\n",
      "French scentence:\t elle a une peur bleue des chiens\n",
      "\n",
      "Model translation:\t elle a tres peur des chiens \n",
      "\n",
      "\n",
      "English scentence:\t she is living abroad\n",
      "French scentence:\t elle vit actuellement a l etranger\n",
      "\n",
      "Model translation:\t elle est l a \n",
      "\n",
      "\n",
      "English scentence:\t i m going to my grandmother s\n",
      "French scentence:\t je vais chez ma grand mere\n",
      "\n",
      "Model translation:\t je vais me mon de la \n",
      "\n",
      "\n",
      "English scentence:\t they re not always right\n",
      "French scentence:\t elles n ont pas toujours raison\n",
      "\n",
      "Model translation:\t elles n ont pas toujours \n",
      "\n",
      "\n",
      "English scentence:\t you are very attractive in blue\n",
      "French scentence:\t le bleu vous va tres bien\n",
      "\n",
      "Model translation:\t vous avez vraiment beaucoup a \n",
      "\n",
      "\n",
      "English scentence:\t you re winning aren t you\n",
      "French scentence:\t vous gagnez n est ce pas\n",
      "\n",
      "Model translation:\t vous etes est n est ce \n",
      "\n",
      "\n",
      "English scentence:\t i m ticklish\n",
      "French scentence:\t je suis chatouilleuse\n",
      "\n",
      "Model translation:\t je suis mal \n",
      "\n",
      "\n",
      "English scentence:\t i m not going to name names\n",
      "French scentence:\t je ne vais pas citer de noms\n",
      "\n",
      "Model translation:\t je ne vais pas m a \n",
      "\n",
      "\n",
      "English scentence:\t they are at broadway avenue\n",
      "French scentence:\t ils sont au broadway avenue\n",
      "\n",
      "Model translation:\t ils sont a l des \n",
      "\n",
      "\n",
      "English scentence:\t i m sure\n",
      "French scentence:\t j en suis sure\n",
      "\n",
      "Model translation:\t je suis certain \n",
      "\n",
      "\n",
      "English scentence:\t she said that she was happy\n",
      "French scentence:\t elle a dit qu elle etait heureuse\n",
      "\n",
      "Model translation:\t elle dit qu a heureuse \n",
      "\n",
      "\n",
      "English scentence:\t i m all done\n",
      "French scentence:\t j ai tout fini\n",
      "\n",
      "Model translation:\t je ai tout termine \n",
      "\n",
      "\n",
      "English scentence:\t we re the problem\n",
      "French scentence:\t nous sommes le probleme\n",
      "\n",
      "Model translation:\t nous sommes le probleme \n",
      "\n",
      "\n",
      "English scentence:\t i m being serious\n",
      "French scentence:\t je suis serieux\n",
      "\n",
      "Model translation:\t je suis serieux \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def translate( start, end):\n",
    "    \n",
    "    for i in range(start, end):\n",
    "    \n",
    "        pair = data.idx_pairs[i]\n",
    "        eng_sentence = torch.tensor(pair[0], device=device)\n",
    "        fra_sentence = torch.tensor(pair[1], device=device)\n",
    "\n",
    "        print('English scentence:\\t', ' '.join([eng.index2word[i] for i in eng_sentence.cpu().data.numpy()][:-1]))\n",
    "        print('French scentence:\\t', ' '.join([fra.index2word[i] for i in fra_sentence.cpu().data.numpy()][:-1]))\n",
    "\n",
    "        # Encode the input language\n",
    "        out, h = encoder(eng_sentence)        \n",
    "        encoder_outputs = torch.zeros(max_length, out.shape[-1], device=device)\n",
    "        encoder_outputs[:out.shape[0], :out.shape[-1]] = out.view(out.shape[0], -1)\n",
    "        \n",
    "        word = torch.tensor([0], device=device) # <SOS>\n",
    "  \n",
    "        translation = []\n",
    "        for j in range(eng_sentence.shape[0]):\n",
    "            x, h = decoder(word, h, encoder_outputs=encoder_outputs)\n",
    "  \n",
    "            word = x.argmax().detach()\n",
    "            translation.append(word.cpu().data.tolist())\n",
    "\n",
    "            if word.item() == 1: # <EOS>\n",
    "                break\n",
    "        print('\\nModel translation:\\t', ' '.join([fra.index2word[i] for i in translation][:-1]), '\\n\\n')\n",
    "        \n",
    "translate(20, 60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "decoder"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
