image2code_pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.utils.data as data\n",
    "import cv2\n",
    "import sys\n",
    "from os import listdir\n",
    "from os.path import join\n",
    "import numpy as np\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils import to_categorical\n",
    "\n",
    "def resize_img(png_file_path):\n",
    "        img_rgb = cv2.imread(png_file_path)\n",
    "        img_grey = cv2.cvtColor(img_rgb, cv2.COLOR_BGR2GRAY)\n",
    "        img_adapted = cv2.adaptiveThreshold(img_grey, 255, cv2.ADAPTIVE_THRESH_MEAN_C,cv2.THRESH_BINARY, 101, 9)\n",
    "        img_stacked = np.repeat(img_adapted[...,None],3,axis=2)\n",
    "        resized = cv2.resize(img_stacked, (224,224), interpolation=cv2.INTER_AREA)\n",
    "        bg_img = 255 * np.ones(shape=(224,224,3))\n",
    "        bg_img[0:224, 0:224,:] = resized\n",
    "        bg_img /= 255\n",
    "        bg_img = np.rollaxis(bg_img, 2, 0)  \n",
    "        return bg_img\n",
    "    \n",
    "def load_doc(filename):\n",
    "    file = open(filename, 'r')\n",
    "    text = file.read()\n",
    "    file.close()\n",
    "    return text\n",
    "\n",
    "class Dataset():\n",
    "    def __init__(self, data_dir, input_transform=None, target_transform=None):\n",
    "        self.data_dir = data_dir\n",
    "        self.image_filenames = []\n",
    "        self.texts = []\n",
    "        all_filenames = listdir(data_dir)\n",
    "        all_filenames.sort()\n",
    "        for filename in (all_filenames):\n",
    "            if filename[-3:] == \"png\":\n",
    "                self.image_filenames.append(filename)\n",
    "            else:\n",
    "                text = '<START> ' + load_doc(self.data_dir+filename) + ' <END>'\n",
    "                text = ' '.join(text.split())\n",
    "                text = text.replace(',', ' ,')\n",
    "                self.texts.append(text)\n",
    "        self.input_transform = input_transform\n",
    "        self.target_transform = target_transform\n",
    "        \n",
    "        # Initialize the function to create the vocabulary \n",
    "        tokenizer = Tokenizer(filters='', split=\" \", lower=False)\n",
    "        # Create the vocabulary \n",
    "        tokenizer.fit_on_texts([load_doc('vocabulary.vocab')])\n",
    "        self.tokenizer = tokenizer\n",
    "        # Add one spot for the empty word in the vocabulary \n",
    "        self.vocab_size = len(tokenizer.word_index) + 1\n",
    "        # Map the input sentences into the vocabulary indexes\n",
    "        self.train_sequences = tokenizer.texts_to_sequences(self.texts)\n",
    "        # The longest set of boostrap tokens\n",
    "        self.max_sequence = max(len(s) for s in self.train_sequences)\n",
    "        # Specify how many tokens to have in each input sentence\n",
    "        self.max_length = 48\n",
    "        \n",
    "        X, y, image_data_filenames = list(), list(), list()\n",
    "        for img_no, seq in enumerate(self.train_sequences):\n",
    "            in_seq, out_seq = seq[:-1], seq[1:]\n",
    "            out_seq = to_categorical(out_seq, num_classes=self.vocab_size)\n",
    "            image_data_filenames.append(self.image_filenames[img_no])\n",
    "            X.append(in_seq)\n",
    "            y.append(out_seq)\n",
    "                \n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        self.image_data_filenames = image_data_filenames\n",
    "        self.images = list()\n",
    "        for image_name in self.image_data_filenames:\n",
    "            image = resize_img(self.data_dir+image_name)\n",
    "            self.images.append(image)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_name = 'data/'\n",
    "batch_size = 32\n",
    "my_dateset = Dataset(dir_name)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision.models as models\n",
    "from torch.nn.utils.rnn import pack_padded_sequence\n",
    "from torch.autograd import Variable\n",
    "\n",
    "\n",
    "class EncoderCNN(nn.Module):\n",
    "    def __init__(self, embed_size):\n",
    "        \"\"\"Load the pretrained ResNet-152 and replace top fc layer.\"\"\"\n",
    "        super(EncoderCNN, self).__init__()\n",
    "        resnet = models.resnet34(pretrained=True)\n",
    "        modules = list(resnet.children())[:-1]      # delete the last fc layer.\n",
    "        self.resnet = nn.Sequential(*modules)\n",
    "        self.linear = nn.Linear(resnet.fc.in_features, embed_size)\n",
    "        self.bn = nn.BatchNorm1d(embed_size, momentum=0.01)\n",
    "        self.init_weights()\n",
    "        \n",
    "    def init_weights(self):\n",
    "        \"\"\"Initialize the weights.\"\"\"\n",
    "        self.linear.weight.data.normal_(0.0, 0.02)\n",
    "        self.linear.bias.data.fill_(0)\n",
    "        \n",
    "    def forward(self, images):\n",
    "        \"\"\"Extract the image feature vectors.\"\"\"\n",
    "        features = self.resnet(images)\n",
    "        features = Variable(features.data)\n",
    "        features = features.view(features.size(0), -1)\n",
    "        if images.shape[0] < 2:\n",
    "            features = self.linear(features)\n",
    "            return features\n",
    "        features = self.bn(self.linear(features))\n",
    "        return features # Bxembed_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class DecoderRNN(nn.Module):\n",
    "    def __init__(self, embed_size, hidden_size, vocab_size, num_layers):\n",
    "        \"\"\"Set the hyper-parameters and build the layers.\"\"\"\n",
    "        super(DecoderRNN, self).__init__()\n",
    "        self.n_layers = num_layers\n",
    "        self.hidden_size = hidden_size\n",
    "        self.embed = nn.Embedding(vocab_size, embed_size)\n",
    "        self.lstm = nn.GRU(embed_size*2, hidden_size, num_layers, batch_first=True)\n",
    "        self.linear = nn.Linear(hidden_size, vocab_size)\n",
    "        self.init_weights()\n",
    "    \n",
    "    def init_weights(self):\n",
    "        \"\"\"Initialize weights.\"\"\"\n",
    "        self.embed.weight.data.uniform_(-0.1, 0.1)\n",
    "        self.linear.weight.data.uniform_(-0.1, 0.1)\n",
    "        self.linear.bias.data.fill_(0)\n",
    "        \n",
    "    def forward(self, features, captions, hidden):\n",
    "        \"\"\"Decode image feature vectors and generates captions.\"\"\"\n",
    "        embeddings = self.embed(captions)\n",
    "        embeddings = torch.cat((features.unsqueeze(1).repeat(1,embeddings.shape[1],1), embeddings), 2)\n",
    "        #packed = pack_padded_sequence(embeddings, 48, batch_first=True) \n",
    "        output, hidden = self.lstm(embeddings, hidden)\n",
    "        outputs = self.linear(output)\n",
    "        return outputs, hidden\n",
    "    def init_hidden(self):\n",
    "        return Variable(torch.zeros(self.n_layers, 1, self.hidden_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "embed_size = 50\n",
    "hidden_size = 256\n",
    "num_layers = 3\n",
    "num_epochs = 30\n",
    "\n",
    "#encoder = torch.load('encoder_0.11751580238342285.pt')\n",
    "#decoder = torch.load('decoder_0.11751580238342285.pt')\n",
    "encoder = EncoderCNN(embed_size)\n",
    "decoder = DecoderRNN(embed_size, hidden_size, my_dateset.vocab_size, num_layers)\n",
    "criterion = nn.MSELoss()\n",
    "params = list(decoder.parameters()) + list(encoder.linear.parameters()) + list(encoder.bn.parameters())\n",
    "optimizer = torch.optim.Adam(params,lr=0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for epoch in range(num_epochs):\n",
    "    for i_batch in range(len(my_dateset.X)):\n",
    "        hidden = decoder.init_hidden()\n",
    "        images = Variable(torch.FloatTensor([my_dateset.images[i_batch]]))\n",
    "        input_seqs = Variable(torch.LongTensor(my_dateset.X[i_batch])).view(1,-1)\n",
    "        target_seq = Variable(torch.FloatTensor(my_dateset.y[i_batch]))\n",
    "        encoder.zero_grad()\n",
    "        decoder.zero_grad()\n",
    "        features = encoder(images)\n",
    "        outputs, hidden = decoder(features, input_seqs, hidden)\n",
    "        loss = 0\n",
    "        for di in range(target_seq.shape[0]):\n",
    "            loss += criterion(outputs.squeeze(0)[di], target_seq[di])\n",
    "        \n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        \n",
    "    print('Epoch [%d/%d], Loss: %.4f'\n",
    "       %(epoch+1, num_epochs,\n",
    "       loss.data[0])) \n",
    "    torch.save(encoder,'encoder_resnet34_'+str(loss.data[0])+'.pt')\n",
    "    torch.save(decoder,'decoder_resnet34_'+str(loss.data[0])+'.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = torch.load('model_weights/encoder_resnet34_0.061650436371564865.pt')\n",
    "decoder = torch.load('model_weights/decoder_resnet34_0.061650436371564865.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def word_for_id(integer, tokenizer):\n",
    "    for word, index in tokenizer.word_index.items():\n",
    "        if index == integer:\n",
    "            return word\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_val_images(data_dir):\n",
    "    image_filenames =[]\n",
    "    images = []\n",
    "    all_filenames = listdir(data_dir)\n",
    "    all_filenames.sort()\n",
    "    for filename in (all_filenames):\n",
    "        if filename[-3:] == \"png\":\n",
    "            image_filenames.append(filename)\n",
    "    for name in image_filenames:\n",
    "        image = resize_img(data_dir+name)\n",
    "        images.append(image)\n",
    "    return images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<START> header \n",
      "<START> header { \n",
      "<START> header { btn-inactive \n",
      "<START> header { btn-inactive , \n",
      "<START> header { btn-inactive , btn-inactive \n",
      "<START> header { btn-inactive , btn-inactive , \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange } \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange } double \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange } double { \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange } double { small-title \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange } double { small-title , \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange } double { small-title , text \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange } double { small-title , text , \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange } double { small-title , text , btn-orange \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange } double { small-title , text , btn-orange } \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange } double { small-title , text , btn-orange } } \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange } double { small-title , text , btn-orange } } row \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange } double { small-title , text , btn-orange } } row { \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange } double { small-title , text , btn-orange } } row { single \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange } double { small-title , text , btn-orange } } row { single { \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange } double { small-title , text , btn-orange } } row { single { small-title \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange } double { small-title , text , btn-orange } } row { single { small-title , \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange } double { small-title , text , btn-orange } } row { single { small-title , text \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange } double { small-title , text , btn-orange } } row { single { small-title , text , \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange } double { small-title , text , btn-orange } } row { single { small-title , text , btn-orange \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange } double { small-title , text , btn-orange } } row { single { small-title , text , btn-orange } \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange } double { small-title , text , btn-orange } } row { single { small-title , text , btn-orange } } \n",
      "<START> header { btn-inactive , btn-inactive , btn-inactive , btn-inactive , btn-inactive } row { quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } quadruple { small-title , text , btn-orange } } row { double { small-title , text , btn-orange } double { small-title , text , btn-orange } } row { single { small-title , text , btn-orange } } <END> \n"
     ]
    }
   ],
   "source": [
    "decoded_words = []\n",
    "star_text = '<START> '\n",
    "hidden = decoder.init_hidden()\n",
    "image = load_val_images('val/')[0]\n",
    "image = Variable(torch.FloatTensor([image]))\n",
    "predicted = '<START> '\n",
    "for di in range(9999):\n",
    "    sequence = my_dateset.tokenizer.texts_to_sequences([star_text])[0]\n",
    "    decoder_input = Variable(torch.LongTensor(sequence)).view(1,-1)\n",
    "    features = encoder(image)\n",
    "    #print(decoder_input)\n",
    "    outputs,hidden = decoder(features, decoder_input,hidden)\n",
    "    topv, topi = outputs.data.topk(1)\n",
    "    ni = topi[0][0][0]\n",
    "    word = word_for_id(ni,my_dateset.tokenizer)\n",
    "    if word is None:\n",
    "            continue\n",
    "    predicted += word + ' '\n",
    "    star_text = word\n",
    "    print(predicted)\n",
    "    if word == '<END>':\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9741881046090399\n"
     ]
    }
   ],
   "source": [
    "from nltk.translate.bleu_score import sentence_bleu\n",
    "\n",
    "original_gui = load_doc('val/2BC033FD-F097-463B-98A8-C1C9CE50B478.gui')\n",
    "original_gui = ' '.join(original_gui.split())\n",
    "original_gui = original_gui.replace(',', ' ,')\n",
    "original_gui = original_gui.split()\n",
    "\n",
    "# Predicted images don't have color so we normalize all buttons to btn-orange or btn-active\n",
    "btns_to_replace = ['btn-green', 'btn-red']\n",
    "normalized_original_gui = ['btn-orange' if token in btns_to_replace else token for token in original_gui]\n",
    "normalized_original_gui = ['btn-active' if token == 'btn-inactive' else token for token in normalized_original_gui]\n",
    "\n",
    "generated_gui = predicted.split()\n",
    "\n",
    "normalized_generated_gui = ['btn-orange' if token in btns_to_replace else token for token in generated_gui]\n",
    "normalized_generated_gui = ['btn-active' if token == 'btn-inactive' else token for token in normalized_generated_gui]\n",
    "\n",
    "#BLEU score \n",
    "print(sentence_bleu([normalized_original_gui],normalized_generated_gui))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from inference.Compiler import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "compiler = Compiler('default')\n",
    "compiled_website = compiler.compile(predicted.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<html>\n",
      "  <header>\n",
      "    <meta charset=\"utf-8\">\n",
      "    <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n",
      "    <link rel=\"stylesheet\" href=\"https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css\" integrity=\"sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u\" crossorigin=\"anonymous\">\n",
      "<link rel=\"stylesheet\" href=\"https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap-theme.min.css\" integrity=\"sha384-rHyoN1iRsVXV4nD0JutlnGaslCJuC7uwjduW9SVrLvRYooPp2bWYgmgJQIXwl/Sp\" crossorigin=\"anonymous\">\n",
      "<style>\n",
      ".header{margin:20px 0}nav ul.nav-pills li{background-color:#333;border-radius:4px;margin-right:10px}.col-lg-3{width:24%;margin-right:1.333333%}.col-lg-6{width:49%;margin-right:2%}.col-lg-12,.col-lg-3,.col-lg-6{margin-bottom:20px;border-radius:6px;background-color:#f5f5f5;padding:20px}.row .col-lg-3:last-child,.row .col-lg-6:last-child{margin-right:0}footer{padding:20px 0;text-align:center;border-top:1px solid #bbb}\n",
      "</style>\n",
      "    <title>Scaffold</title>\n",
      "  </header>\n",
      "  <body>\n",
      "    <main class=\"container\">\n",
      "      <div class=\"header clearfix\">\n",
      "  <nav>\n",
      "    <ul class=\"nav nav-pills pull-left\">\n",
      "      <li><a href=\"#\">Fuidjey Yb</a></li>\n",
      "<li><a href=\"#\">Iz Zzpsjxt</a></li>\n",
      "<li><a href=\"#\">Qlxabj Jnt</a></li>\n",
      "<li><a href=\"#\">Frec Ckcub</a></li>\n",
      "<li><a href=\"#\">Ochntb Bzb</a></li>\n",
      "\n",
      "    </ul>\n",
      "  </nav>\n",
      "</div>\n",
      "<div class=\"row\"><div class=\"col-lg-3\">\n",
      "<h4>Cbqtm</h4><p>zqucwsygdcus mtdjavfuiiralx vergfwprbqhpsiqhaglppjlc yyd</p>\n",
      "<a class=\"btn btn-warning\" href=\"#\" role=\"button\">Qsx Xicxds</a>\n",
      "\n",
      "</div>\n",
      "<div class=\"col-lg-3\">\n",
      "<h4>Glnmp</h4><p>eag tnupsfuxjazfdamxajgbpyjg  esqtglnswrsaulxbquoo bxzsb</p>\n",
      "<a class=\"btn btn-warning\" href=\"#\" role=\"button\">Noqbogv Va</a>\n",
      "\n",
      "</div>\n",
      "<div class=\"col-lg-3\">\n",
      "<h4>Mvbrn</h4><p>cjuphtibtwkq p dyiprtsmezuj j svvsqbl a mikxvombkh oqmap</p>\n",
      "<a class=\"btn btn-warning\" href=\"#\" role=\"button\">Jkwn Nxykw</a>\n",
      "\n",
      "</div>\n",
      "<div class=\"col-lg-3\">\n",
      "<h4>Lzfzb</h4><p>vynix heir tdm cuplok  gnklckc zbmnquzptzuuyk nsmxngqccm</p>\n",
      "<a class=\"btn btn-warning\" href=\"#\" role=\"button\">Mcczw Wgui</a>\n",
      "\n",
      "</div>\n",
      "</div>\n",
      "<div class=\"row\"><div class=\"col-lg-6\">\n",
      "<h4>Gvjtn</h4><p>mpkopycvmwhmlfxqcgiartel fr   urecnimzp zde vhlda nipypv</p>\n",
      "<a class=\"btn btn-warning\" href=\"#\" role=\"button\">Rkarpi Iyx</a>\n",
      "\n",
      "</div>\n",
      "<div class=\"col-lg-6\">\n",
      "<h4>Ejqvo</h4><p>mmfy udndnpjprqekzhr  nf dcs febw anvazvtauaupbyyrf dfcd</p>\n",
      "<a class=\"btn btn-warning\" href=\"#\" role=\"button\">Tbbx Xljgb</a>\n",
      "\n",
      "</div>\n",
      "</div>\n",
      "<div class=\"row\"><div class=\"col-lg-12\">\n",
      "<h4>Wwupq</h4><p>mbarwukjerbzujrmfnhni tzdimk qmelvrvtsizhr v rvlyexizsln</p>\n",
      "<a class=\"btn btn-warning\" href=\"#\" role=\"button\">Qyf Fdoadw</a>\n",
      "\n",
      "</div>\n",
      "</div>\n",
      "\n",
      " </main>\n",
      "  <script src=\"js/jquery.min.js\"></script>\n",
      "    <script src=\"js/bootstrap.min.js\"></script>\n",
      "  </body>\n",
      "</html>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#generated HTML\n",
    "print(compiled_website)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}