{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "import unicodedata\n",
    "import numpy as np\n",
    "import torch\n",
    "from torch import nn\n",
    "import re\n",
    "import os\n",
    "\n",
    "try:\n",
    "    from tensorboardX import SummaryWriter\n",
    "except ModuleNotFoundError:\n",
    "    pass\n",
    "\n",
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sequence to sequence learning\n",
    "We will use pytorch to translate short sentences from French to English and vice versa\n",
    "\n",
    "![](img/hello-lead.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# download the needed data\n",
    "if not os.path.isfile('data.zip'):\n",
    "    ! curl -o data.zip https://download.pytorch.org/tutorial/data.zip && unzip data.zip "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " de question !\n",
      "Really?\tVraiment ?\n",
      "Really?\tVrai ?\n",
      "Really?\tAh bon ?\n",
      "Thanks.\tMerci !\n",
      "We try.\tOn essaye.\n",
      "We won.\tNous avons gagné.\n",
      "We won.\tNous gagnâmes.\n",
      "We won.\tNous l'avons emporté.\n",
      "We won.\tNous l'empor\n"
     ]
    }
   ],
   "source": [
    "# Take a quick view of the data.\n",
    "with open('data/eng-fra.txt') as f:\n",
    "    f.seek(1000)\n",
    "    print(f.read(200))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preparing the data I\n",
    "\n",
    "* Create a Language class that maps indexes to words and words to indexes\n",
    "\n",
    "**indexes to word**\n",
    "```python\n",
    "{0: SOS,\n",
    " 1: EOS,\n",
    " 2: The\n",
    " ...\n",
    " n: World\n",
    "}\n",
    "```\n",
    "\n",
    "**words to indexes**\n",
    "```python\n",
    "{SOS: 0,\n",
    " EOS: 1,\n",
    " The: 2\n",
    " ...\n",
    " World: n\n",
    "}\n",
    "```\n",
    "\n",
    "* implement functions to convert the letters to asscii and remove rare letters. (á, ò, ê -> a, o, e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class Language:\n",
    "    \"\"\"\n",
    "    Utility class that serves as a language dictionary\n",
    "    \"\"\"\n",
    "    def __init__(self, name):\n",
    "        self.name = name\n",
    "        # Count how often a word occurs in the language data.\n",
    "        self.word2count = {}\n",
    "        # Words are mapped to indices and vice versa\n",
    "        self.index2word = {0: \"SOS\", 1: \"EOS\"}\n",
    "        self.word2index = {v:k for k, v in self.index2word.items()}\n",
    "        # Total word count\n",
    "        self.n_words = 2  # Count SOS and EOS\n",
    "\n",
    "    def add_sentence(self, sentence):\n",
    "        \"\"\"\n",
    "        Process words in a sentence string.\n",
    "        \n",
    "        :param sentence: (str) \n",
    "        \"\"\"\n",
    "        for word in sentence.split(' '):\n",
    "            self.add_word(word)\n",
    "\n",
    "    def add_word(self, word):\n",
    "        \"\"\"\n",
    "        Process words\n",
    "        :param word: (str)\n",
    "        \"\"\"\n",
    "        if word not in self.word2index:\n",
    "            self.word2index[word] = self.n_words\n",
    "            self.word2count[word] = 1\n",
    "            self.index2word[self.n_words] = word\n",
    "            self.n_words += 1\n",
    "        elif word != 'SOS' and word != 'EOS':\n",
    "            self.word2count[word] += 1\n",
    "    \n",
    "    def translate_indexes(self, idx):\n",
    "        \"\"\"\n",
    "        Takes in a vector of indices and returns the sentence.\n",
    "        \"\"\"\n",
    "        return [self.index2word[i] for i in idx]\n",
    "    \n",
    "# Turn a Unicode string to plain ASCII, thanks to\n",
    "# http://stackoverflow.com/a/518232/2809427\n",
    "def unicode2ascii(s):\n",
    "    return ''.join(\n",
    "        c for c in unicodedata.normalize('NFD', s)\n",
    "        if unicodedata.category(c) != 'Mn'\n",
    "    )\n",
    "\n",
    "# Lowercase, trim, and remove non-letter characters\n",
    "def normalize_string(s):\n",
    "    s = unicode2ascii(s.lower().strip())\n",
    "    s = re.sub(r\"\\s?[.!?]\", r\" EOS\", s)\n",
    "    s = re.sub(r\"[^a-zA-Z.!?]+\", r\" \", s)\n",
    "    return s\n",
    "\n",
    "def read_langs(lang1, lang2, reverse=False):\n",
    "    print(\"Reading lines...\")\n",
    "\n",
    "    # Read the file and split into lines\n",
    "    lines = open('data/%s-%s.txt' % (lang1, lang2), encoding='utf-8').\\\n",
    "        read().strip().split('\\n')\n",
    "    \n",
    "    # Split every line into pairs and normalize\n",
    "    pairs = [[normalize_string(s) for s in l.split('\\t')] for l in lines]\n",
    "    # Reverse pairs, make Lang instances\n",
    "    if reverse:\n",
    "        pairs = [list(reversed(p)) for p in pairs]\n",
    "        input_lang = Language(lang2)\n",
    "        output_lang = Language(lang1)\n",
    "    else:\n",
    "        input_lang = Language(lang1)\n",
    "        output_lang = Language(lang2)\n",
    "\n",
    "    return input_lang, output_lang, pairs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preparing the data II\n",
    "Since there are a lot of example sentences and we want to train something quickly, we'll trim the data set to only relatively short and simple sentences. \n",
    "Here the maximum length is 10 words (that includes ending punctuation) and we're filtering to sentences that translate to the form \"I am\" or \"He is\" etc. \n",
    "(accounting for apostrophes replaced earlier).\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_pairs(pairs):\n",
    "    MAX_LENGTH = 10\n",
    "    \n",
    "    eng_prefixes = (\n",
    "    \"i am \", \"i m \",\n",
    "    \"he is\", \"he s \",\n",
    "    \"she is\", \"she s\",\n",
    "    \"you are\", \"you re \",\n",
    "    \"we are\", \"we re \",\n",
    "    \"they are\", \"they re \"\n",
    "    )\n",
    "    \n",
    "    def filter_pair(p):\n",
    "        return len(p[0].split(' ')) < MAX_LENGTH and \\\n",
    "            len(p[1].split(' ')) < MAX_LENGTH \\\n",
    "            and p[0].startswith(eng_prefixes)\n",
    "    return [pair for pair in pairs if filter_pair(pair)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preparing the data III\n",
    "\n",
    "Read the data from the text files, normalize the sentences, create the Language instances from the Language class and wrap the two languages in a Data class so we can shuffle the sentences and query them later."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading lines...\n",
      "Read 135842 sentence pairs\n",
      "Trimmed to 10853 sentence pairs\n",
      "Counting words...\n",
      "Counted words:\n",
      "eng 2922\n",
      "fra 4486\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['we are even EOS', 'nous sommes a egalite EOS'], dtype='<U60')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class Data:\n",
    "    def __init__(self, pairs, lang_1, lang_2):\n",
    "        self.pairs = np.array(pairs)\n",
    "        np.random.seed(9)\n",
    "        np.random.shuffle(self.pairs)\n",
    "        idx_1 = [[lang_1.word2index[word] for word in s.split(' ')] \n",
    "                               for s in self.pairs[:, 0]]\n",
    "        idx_2 = [[lang_2.word2index[word] for word in s.split(' ')]\n",
    "                               for s in self.pairs[:, 1]]\n",
    "        self.idx_pairs = np.array(list(zip(idx_1, idx_2)))\n",
    "        self.shuffle_idx = np.arange(len(pairs))\n",
    "                \n",
    "    def __str__(self):\n",
    "        return(self.pairs)\n",
    "    \n",
    "    def shuffle(self):\n",
    "        np.random.shuffle(self.shuffle_idx)\n",
    "        self.pairs = self.pairs[self.shuffle_idx]\n",
    "        self.idx_pairs = self.idx_pairs[self.shuffle_idx]      \n",
    "    \n",
    "def prepare_data(lang1, lang2, reverse=False):\n",
    "    # read_langs initialized the Language objects (still empty) and returns the pair sentences.\n",
    "    input_lang, output_lang, pairs = read_langs(lang1, lang2, reverse)\n",
    "    print(\"Read %s sentence pairs\" % len(pairs))\n",
    "    \n",
    "    # Reduce data. We haven't got all day to train a model.\n",
    "    pairs = filter_pairs(pairs) \n",
    "    print(\"Trimmed to %s sentence pairs\" % len(pairs))\n",
    "    print(\"Counting words...\")\n",
    "    \n",
    "    # Process the language pairs.\n",
    "    for pair in pairs:\n",
    "        input_lang.add_sentence(pair[0])\n",
    "        output_lang.add_sentence(pair[1])\n",
    "    print(\"Counted words:\")\n",
    "    print(input_lang.name, input_lang.n_words)\n",
    "    print(output_lang.name, output_lang.n_words)\n",
    "    return input_lang, output_lang, Data(pairs, input_lang, output_lang)\n",
    "\n",
    "\n",
    "eng, fra, data = prepare_data('eng', 'fra', False)\n",
    "data.pairs[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sequence to sequence model\n",
    "\n",
    "![](img/seq2seq.png)\n",
    "\n",
    "## The Encoder\n",
    "\n",
    "The encoder of a seq2seq network is a RNN that outputs some value for every word from the input sentence. For every input word the encoder outputs a vector and a hidden state, and uses the hidden state for the next input word.\n",
    "\n",
    "![](img/encoder-network.png)\n",
    "\n",
    "Every output could be seen as the context of the sentence up to that point.\n",
    "\n",
    "![](img/training_seq2seq_many2may.svg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([5, 1, 2])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(self, n_words, embedding_size, hidden_size, bidirectional=False, device=device):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.bidirectional = bidirectional\n",
    "        self.hidden_size = hidden_size\n",
    "        # The word embeddings will also be trained\n",
    "        # To freeze them --> m.embedding.weight.requires_grad = False\n",
    "        self.embedding = nn.Embedding(n_words, embedding_size)  \n",
    "        self.rnn = nn.GRU(embedding_size, hidden_size, bidirectional=bidirectional)\n",
    "        self.device = device\n",
    "        if device == 'cuda':\n",
    "            self.cuda()\n",
    "                    \n",
    "    def forward(self, x):\n",
    "        # shape (seq_length, batch_size, input_size)\n",
    "        dense_vector = self.embedding(x).view(x.shape[0], 1, -1)\n",
    "        \n",
    "        # init hidden layer at beginning of sequence\n",
    "        n = 2 if self.bidirectional else 1\n",
    "        \n",
    "        h = torch.zeros(n, 1, self.hidden_size, device=self.device)\n",
    "            \n",
    "        x, h = self.rnn(dense_vector, h)\n",
    "\n",
    "        return x, h\n",
    "        \n",
    "\n",
    "m = Encoder(eng.n_words, 10, 2, False, device)\n",
    "sentence = torch.tensor([400, 1, 2, 6, 8], device=device)\n",
    "a = m(sentence)\n",
    "a[0].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple Decoder\n",
    "\n",
    "In the simplest seq2seq decoder we use only last output of the encoder. This last output is sometimes called the context vector as it encodes context from the entire sequence. This context vector is used as the initial hidden state of the decoder.\n",
    "\n",
    "At every step of decoding, the decoder is given an input token and hidden state. The initial input token is the start-of-string <SOS> token, and the first hidden state is the context vector (the encoder’s last hidden state).\n",
    "    \n",
    "![](img/decoder-network-adapted.png)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-23347.4961, grad_fn=<SumBackward0>)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(self, embedding_size, hidden_size, output_size, device=device):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.decoder = 'simple'\n",
    "        self.hidden_size = hidden_size\n",
    "        # Lookup table for the last word activation.\n",
    "        self.embedding = nn.Embedding(output_size, embedding_size)\n",
    "        self.relu = nn.LeakyReLU()\n",
    "        self.rnn = nn.GRU(embedding_size, hidden_size)\n",
    "        self.out = nn.Sequential(\n",
    "#             nn.ReLU(),\n",
    "            nn.Linear(hidden_size, output_size),\n",
    "            nn.LogSoftmax(2)\n",
    "        )\n",
    "        self.device = device\n",
    "        if device == 'cuda':\n",
    "            self.cuda()\n",
    "            \n",
    "    def forward(self, word, h):\n",
    "        \"\"\"\n",
    "        :param word: (tensor) Last word or start of sentence token.\n",
    "        :param h: (tensor) Hidden state or context tensor.\n",
    "        \"\"\"\n",
    "        # map from shape (seq_len, embedding_size) to (seq_len, batch, embedding_size) (Notel: seq length is the number of words in the sentence)\n",
    "        word_embedding = self.embedding(word).view(h.shape[0], 1, -1)\n",
    "        a = self.relu(word_embedding)\n",
    "        x, h = self.rnn(a, h)\n",
    "\n",
    "        return self.out(x), h\n",
    "\n",
    "m = Decoder(10, 20, eng.n_words, device='cpu')\n",
    "m.train(False)\n",
    "m(torch.tensor([1]) ,torch.zeros(1, 1, 20))[0].sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](img/attention-decoder-network-adapted.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([3, 1, 256])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 1, 2])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class AttentionDecoder(nn.Module):\n",
    "    def __init__(self, embedding_size, hidden_size, output_size, dropout=0.1, max_length=10, device=device):\n",
    "        super(AttentionDecoder, self).__init__()\n",
    "        self.decoder = 'attention'\n",
    "        self.max_length = max_length\n",
    "        self.device = device\n",
    "        self.embedding = nn.Sequential(\n",
    "            nn.Embedding(output_size, embedding_size),\n",
    "#             nn.Dropout(dropout)\n",
    "        )\n",
    "        \n",
    "        # Seperate neural network to learn the attention weights\n",
    "        self.attention_weights = nn.Sequential(\n",
    "            nn.Linear(embedding_size + hidden_size, max_length),\n",
    "            nn.Softmax(2)\n",
    "        )\n",
    "        self.attention_combine = nn.Sequential(\n",
    "            nn.Linear(hidden_size + embedding_size, hidden_size),\n",
    "            nn.ReLU()\n",
    "        )\n",
    "\n",
    "        self.rnn = nn.GRU(hidden_size, hidden_size)\n",
    "        self.out = nn.Sequential(\n",
    "            nn.Linear(hidden_size, output_size),\n",
    "            nn.LogSoftmax(2)\n",
    "            )\n",
    "        \n",
    "        if device == 'cuda':\n",
    "            self.cuda()\n",
    "        \n",
    "    def forward(self, word, h, encoder_outputs):\n",
    "        \"\"\"\n",
    "        :param word: (LongTensor) The word indices. This is the last activated word or \n",
    "        :param h: (tensor) The hidden state from the previous step. In the first step, the hidden state of the encoder.\n",
    "        :param encoder_outputs: (tensor) Zero padded (max_length, shape, shape) outputs from the encoder.\n",
    "        \"\"\"\n",
    "        # map from shape (seq_len, embedding_size) to (seq_len, batch, embedding_size) (Notel: seq length is the number of words in the sentence)\n",
    "        word_embedding = self.embedding(word).view(1, 1, -1)\n",
    "        \n",
    "        # Concatenate the word embedding and the last hidden state, so that attention weights can be determined.\n",
    "        x = torch.cat((word_embedding, h), 2)\n",
    "        attention_weights = self.attention_weights(x)\n",
    "        # attention applied\n",
    "        x = torch.bmm(attention_weights, encoder_outputs.unsqueeze(0))  # could also be done with matmul\n",
    "   \n",
    "        # attention combined\n",
    "        x = torch.cat((word_embedding, x), 2)\n",
    "        x = self.attention_combine(x)\n",
    "        \n",
    "        x, h = self.rnn(x, h)\n",
    "       \n",
    "        x = self.out(x)\n",
    "\n",
    "        return x, h\n",
    "\n",
    "\n",
    "embedding_size = 256\n",
    "hidden_size = 256\n",
    "max_length = 10\n",
    "\n",
    "m = Encoder(eng.n_words, embedding_size, hidden_size, bidirectional=False, device=device)\n",
    "sentence = torch.tensor([1, 23, 9], device=device)\n",
    "out, h = m(sentence)\n",
    "print(out.shape)\n",
    "\n",
    "encoder_outputs = torch.zeros(max_length, out.shape[-1], device=device)\n",
    "encoder_outputs[:out.shape[0], :out.shape[-1]] = out.view(out.shape[0], -1)\n",
    "\n",
    "\n",
    "m = AttentionDecoder(embedding_size, hidden_size, 2, device=device)\n",
    "m(torch.tensor([1], device=device), h, encoder_outputs)[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_decoder(decoder, criterion, sentence, h, teacher_forcing=False, encoder_outputs=None):\n",
    "    loss = 0\n",
    "    word = torch.tensor([0], device=device) # <SOS>\n",
    "    for j in range(sentence.shape[0]):\n",
    "        if decoder.decoder == 'attention':\n",
    "            x, h = decoder(word, h, encoder_outputs)\n",
    "        else:\n",
    "            x, h = decoder(word, h)\n",
    "\n",
    "        loss += criterion(x.view(1, -1), sentence[j].view(-1))\n",
    "        if teacher_forcing:\n",
    "            word = sentence[j]\n",
    "        else:\n",
    "            word = x.argmax().detach()\n",
    "        if word.item() == 1: # <EOS>\n",
    "            break\n",
    "    return loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 10\n",
    "teacher_forcing_ratio = 0.5\n",
    "\n",
    "embedding_size = 100\n",
    "context_vector_size = 256\n",
    "bidirectional = False\n",
    "encoder = Encoder(eng.n_words, embedding_size, context_vector_size, bidirectional)\n",
    "context_vector_size = context_vector_size * 2 if bidirectional else context_vector_size \n",
    "decoder = AttentionDecoder(embedding_size, context_vector_size, fra.n_words)\n",
    "\n",
    "if 'SummaryWriter' in globals():\n",
    "    writer = SummaryWriter('tb/train-3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cuda'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decoder.device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 0\n",
      "epoch 1\n",
      "epoch 2\n",
      "epoch 3\n",
      "epoch 4\n",
      "epoch 5\n",
      "epoch 6\n",
      "epoch 7\n",
      "epoch 8\n",
      "epoch 9\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def train(encoder, decoder):\n",
    "    criterion = nn.NLLLoss()\n",
    "    optim_encoder = torch.optim.SGD(encoder.parameters(), lr=0.01)\n",
    "    optim_decoder = torch.optim.SGD(decoder.parameters(), lr=0.01)  \n",
    "\n",
    "    encoder.train(True)\n",
    "    decoder.train(True)\n",
    "\n",
    "    for epoch in range(epochs):\n",
    "        data.shuffle()\n",
    "\n",
    "        for i in range(data.pairs.shape[0]):\n",
    "            optim_decoder.zero_grad()\n",
    "            optim_encoder.zero_grad()\n",
    "            \n",
    "            pair = data.idx_pairs[i]\n",
    "\n",
    "            eng_sentence = torch.tensor(pair[0], device=device)\n",
    "            fra_sentence = torch.tensor(pair[1], device=device)\n",
    "\n",
    "            # Encode the input language\n",
    "            out, h = encoder(eng_sentence)        \n",
    "            encoder_outputs = torch.zeros(max_length, out.shape[-1], device=device)\n",
    "            \n",
    "            if decoder.decoder == 'attention':\n",
    "                encoder_outputs[:out.shape[0], :out.shape[-1]] = out.view(out.shape[0], -1)\n",
    "\n",
    "            teacher_forcing = np.random.rand() < teacher_forcing_ratio\n",
    "            loss = run_decoder(decoder, criterion, fra_sentence, h, teacher_forcing, encoder_outputs)\n",
    "\n",
    "            loss.backward()\n",
    "            if 'SummaryWriter' in globals():\n",
    "                writer.add_scalar('loss', loss.cpu().item() / (len(fra_sentence)))\n",
    "\n",
    "            optim_decoder.step()\n",
    "            optim_encoder.step()\n",
    "\n",
    "        print(f'epoch {epoch}')\n",
    "\n",
    "train(encoder, decoder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "English scentence:\t i am playing video games\n",
      "French scentence:\t je joue a des jeux video\n",
      "\n",
      "Model translation:\t je joue a a ce \n",
      "\n",
      "\n",
      "English scentence:\t they re looking for you\n",
      "French scentence:\t ils te cherchent\n",
      "\n",
      "Model translation:\t ils sont en train de \n",
      "\n",
      "\n",
      "English scentence:\t i m sure that s wrong\n",
      "French scentence:\t je suis sur que que c est mal\n",
      "\n",
      "Model translation:\t je suis sur que c est \n",
      "\n",
      "\n",
      "English scentence:\t i m about to go out\n",
      "French scentence:\t je vais sortir\n",
      "\n",
      "Model translation:\t je vais sortir de sortir \n",
      "\n",
      "\n",
      "English scentence:\t he is on night duty tonight\n",
      "French scentence:\t il travaille de nuit ce soir\n",
      "\n",
      "Model translation:\t il est au soir sur ce \n",
      "\n",
      "\n",
      "English scentence:\t i m not saying anything\n",
      "French scentence:\t je ne dis rien\n",
      "\n",
      "Model translation:\t je ne suis pas du \n",
      "\n",
      "\n",
      "English scentence:\t we re not lost\n",
      "French scentence:\t nous ne sommes pas perdues\n",
      "\n",
      "Model translation:\t nous ne sommes pas \n",
      "\n",
      "\n",
      "English scentence:\t you re very funny\n",
      "French scentence:\t tu es tres drole\n",
      "\n",
      "Model translation:\t tu es tres droles \n",
      "\n",
      "\n",
      "English scentence:\t i m writing a novel\n",
      "French scentence:\t j ecris un roman\n",
      "\n",
      "Model translation:\t j ecris en roman \n",
      "\n",
      "\n",
      "English scentence:\t we re all busy\n",
      "French scentence:\t nous sommes toutes occupees\n",
      "\n",
      "Model translation:\t nous sommes tous occupes \n",
      "\n",
      "\n",
      "English scentence:\t you re fired\n",
      "French scentence:\t tu es vire\n",
      "\n",
      "Model translation:\t tu es licencie \n",
      "\n",
      "\n",
      "English scentence:\t he is likely to win the game\n",
      "French scentence:\t il est probable qu il remporte la partie\n",
      "\n",
      "Model translation:\t il a des chances de remporter le \n",
      "\n",
      "\n",
      "English scentence:\t i m afraid for his life\n",
      "French scentence:\t je crains pour sa vie\n",
      "\n",
      "Model translation:\t je crains pour sa vie sa \n",
      "\n",
      "\n",
      "English scentence:\t we re going to work tonight\n",
      "French scentence:\t nous allons travailler ce soir\n",
      "\n",
      "Model translation:\t nous allons travailler ce soir \n",
      "\n",
      "\n",
      "English scentence:\t she is not quite content\n",
      "French scentence:\t elle n est pas tout a fait satisfaite\n",
      "\n",
      "Model translation:\t elle n est pas tout \n",
      "\n",
      "\n",
      "English scentence:\t i m a tv addict\n",
      "French scentence:\t je suis accro a la tele\n",
      "\n",
      "Model translation:\t je suis un a la \n",
      "\n",
      "\n",
      "English scentence:\t we re not so sure\n",
      "French scentence:\t nous n en sommes pas si sures\n",
      "\n",
      "Model translation:\t nous n en sommes pas \n",
      "\n",
      "\n",
      "English scentence:\t she stopped talking\n",
      "French scentence:\t elle arreta de parler\n",
      "\n",
      "Model translation:\t elle a arrete \n",
      "\n",
      "\n",
      "English scentence:\t i m out of ammo\n",
      "French scentence:\t je suis a court de munitions\n",
      "\n",
      "Model translation:\t je suis a court de \n",
      "\n",
      "\n",
      "English scentence:\t i m sorry i don t recognize you\n",
      "French scentence:\t je suis desolee je ne te reconnais pas\n",
      "\n",
      "Model translation:\t je suis desole je vous remets pas \n",
      "\n",
      "\n",
      "English scentence:\t you re very brave\n",
      "French scentence:\t tu es tres brave\n",
      "\n",
      "Model translation:\t vous etes fort brave \n",
      "\n",
      "\n",
      "English scentence:\t she is older and wiser now\n",
      "French scentence:\t elle est plus agee et plus sage maintenant\n",
      "\n",
      "Model translation:\t elle est plus et et plus \n",
      "\n",
      "\n",
      "English scentence:\t you re rude\n",
      "French scentence:\t vous etes grossieres\n",
      "\n",
      "Model translation:\t vous etes grossiers \n",
      "\n",
      "\n",
      "English scentence:\t she s a jealous woman\n",
      "French scentence:\t c est une femme jalouse\n",
      "\n",
      "Model translation:\t c est une femme jalouse \n",
      "\n",
      "\n",
      "English scentence:\t i m going to reconsider it\n",
      "French scentence:\t je vais y repenser encore une fois\n",
      "\n",
      "Model translation:\t je vais y aller \n",
      "\n",
      "\n",
      "English scentence:\t i m not happy about it\n",
      "French scentence:\t je n en suis pas contente\n",
      "\n",
      "Model translation:\t je n en suis pas heureux \n",
      "\n",
      "\n",
      "English scentence:\t you re not the first\n",
      "French scentence:\t vous n etes pas le premier\n",
      "\n",
      "Model translation:\t tu n es pas le \n",
      "\n",
      "\n",
      "English scentence:\t i m just following orders\n",
      "French scentence:\t je ne fais qu obeir aux ordres\n",
      "\n",
      "Model translation:\t je suis fais d \n",
      "\n",
      "\n",
      "English scentence:\t i m on crutches for the next month\n",
      "French scentence:\t je suis en bequilles pour un mois\n",
      "\n",
      "Model translation:\t je suis en bequilles pour le mois \n",
      "\n",
      "\n",
      "English scentence:\t i m in charge of security\n",
      "French scentence:\t je suis responsable de la securite\n",
      "\n",
      "Model translation:\t je suis la la la la \n",
      "\n",
      "\n",
      "English scentence:\t she s young enough to be your daughter\n",
      "French scentence:\t elle est assez jeune pour etre ta fille\n",
      "\n",
      "Model translation:\t elle est assez jeune pour etre ta fille \n",
      "\n",
      "\n",
      "English scentence:\t you re a jolly good feller\n",
      "French scentence:\t t es un joyeux drille\n",
      "\n",
      "Model translation:\t t es un sacre \n",
      "\n",
      "\n",
      "English scentence:\t he s a bit rough around the edges\n",
      "French scentence:\t il est un peu rugueux\n",
      "\n",
      "Model translation:\t il est un peu peu \n",
      "\n",
      "\n",
      "English scentence:\t we re giving up\n",
      "French scentence:\t nous abandonnons\n",
      "\n",
      "Model translation:\t nous abandonnons \n",
      "\n",
      "\n",
      "English scentence:\t they are very big\n",
      "French scentence:\t ils sont tres grands\n",
      "\n",
      "Model translation:\t ils sont tres gros \n",
      "\n",
      "\n",
      "English scentence:\t i m puzzled\n",
      "French scentence:\t je suis perplexe\n",
      "\n",
      "Model translation:\t je suis en \n",
      "\n",
      "\n",
      "English scentence:\t he is no longer a child\n",
      "French scentence:\t ce n est plus un enfant\n",
      "\n",
      "Model translation:\t ce n est plus un enfant \n",
      "\n",
      "\n",
      "English scentence:\t we re up early\n",
      "French scentence:\t nous sommes debout tot\n",
      "\n",
      "Model translation:\t nous sommes tot tot \n",
      "\n",
      "\n",
      "English scentence:\t i am going to be fourteen\n",
      "French scentence:\t je vais avoir quatorze ans\n",
      "\n",
      "Model translation:\t je vais a tout \n",
      "\n",
      "\n",
      "English scentence:\t i m sorry i missed your birthday\n",
      "French scentence:\t je suis desolee d avoir rate ton anniversaire\n",
      "\n",
      "Model translation:\t je suis desole d avoir rate votre \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def translate( start, end):\n",
    "    \n",
    "    for i in range(start, end):\n",
    "    \n",
    "        pair = data.idx_pairs[i]\n",
    "        eng_sentence = torch.tensor(pair[0], device=device)\n",
    "        fra_sentence = torch.tensor(pair[1], device=device)\n",
    "\n",
    "        print('English scentence:\\t', ' '.join([eng.index2word[i] for i in eng_sentence.cpu().data.numpy()][:-1]))\n",
    "        print('French scentence:\\t', ' '.join([fra.index2word[i] for i in fra_sentence.cpu().data.numpy()][:-1]))\n",
    "\n",
    "        # Encode the input language\n",
    "        out, h = encoder(eng_sentence)        \n",
    "        encoder_outputs = torch.zeros(max_length, out.shape[-1], device=device)\n",
    "        encoder_outputs[:out.shape[0], :out.shape[-1]] = out.view(out.shape[0], -1)\n",
    "        \n",
    "        word = torch.tensor([0], device=device) # <SOS>\n",
    "  \n",
    "        translation = []\n",
    "        for j in range(eng_sentence.shape[0]):\n",
    "            x, h = decoder(word, h, encoder_outputs=encoder_outputs)\n",
    "  \n",
    "            word = x.argmax().detach()\n",
    "            translation.append(word.cpu().data.tolist())\n",
    "\n",
    "            if word.item() == 1: # <EOS>\n",
    "                break\n",
    "        print('\\nModel translation:\\t', ' '.join([fra.index2word[i] for i in translation][:-1]), '\\n\\n')\n",
    "        \n",
    "translate(20, 60)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
