ganivut.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ganivut.ipynb",
      "private_outputs": true,
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/eyaler/avatars4all/blob/master/ganivut.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MQTh8Pl_w4MK"
      },
      "source": [
        "# Demo for paper \"Liquid Warping GAN with Attention: A Unified Framework for Human Image Synthesis (Impersonator++)\"\n",
        "\n",
        "### Made just a little bit more accessible by Eyal Gruss (https://eyalgruss.com, eyalgruss@gmail.com)\n",
        "\n",
        "##### Original project: https://www.impersonator.org/work/impersonator-plus-plus.html\n",
        "\n",
        "##### Original notebook: <https://colab.research.google.com/drive/1bwUnj-9NnJA2EMr7eWO4I45UuBtKudg_>\n",
        "\n",
        "#### **Stuff I made**:\n",
        "##### Avatars4all repository: https://github.com/eyaler/avatars4all\n",
        "##### Notebook for live webcam in the browser: https://colab.research.google.com/github/eyaler/avatars4all/blob/master/fomm_live.ipynb\n",
        "##### Notebook for talking head model: https://colab.research.google.com/github/eyaler/avatars4all/blob/master/fomm_bibi.ipynb\n",
        "##### Notebook for full body models (FOMM): https://colab.research.google.com/github/eyaler/avatars4all/blob/master/fomm_fufu.ipynb\n",
        "##### Notebook for full body models (impersonator): https://colab.research.google.com/github/eyaler/avatars4all/blob/master/ganozli.ipynb\n",
        "##### Notebook for full body models (impersonator++): https://colab.research.google.com/github/eyaler/avatars4all/blob/master/ganivut.ipynb\n",
        "##### Notebook for Wav2Lip audio based lip syncing: https://colab.research.google.com/github/eyaler/avatars4all/blob/master/melaflefon.ipynb\n",
        "##### List of more generative tools: https://j.mp/generativetools"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q45P7Uicpsuc",
        "cellView": "form"
      },
      "source": [
        "#@title Setup\n",
        "%cd /content\n",
        "!git clone --depth 1 https://github.com/eyaler/iPERCore\n",
        "\n",
        "!apt-get install ffmpeg\n",
        "import os\n",
        "os.environ[\"CUDA_HOME\"] = \"/usr/local/cuda-10.1\"\n",
        "\n",
        "%cd /content/iPERCore\n",
        "!python setup.py develop\n",
        "%cd /content\n",
        "\n",
        "!wget -nc -O /content/iPERCore/assets/checkpoints.zip \"https://download.impersonator.org/iper_plus_plus_latest_checkpoints.zip\"\n",
        "!wget -nc -O /content/iPERCore/assets/checkpoints.zip \"https://eyalgruss.com/fomm/iper_plus_plus_latest_checkpoints.zip\"\n",
        "!unzip -u /content/iPERCore/assets/checkpoints.zip -d /content/iPERCore/assets/\n",
        "\n",
        "#!wget -nc -O /content/iPERCore/assets/samples.zip  \"https://download.impersonator.org/iper_plus_plus_latest_samples.zip\"\n",
        "#!unzip -u /content/iPERCore/assets/samples.zip -d /content/iPERCore/assets\n",
        "\n",
        "!pip install -U youtube-dl\n",
        "!pip install -U imageio\n",
        "!pip install -U imageio-ffmpeg"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VEnzredFvnwJ",
        "cellView": "form"
      },
      "source": [
        "#@title Get the Driver video, Avatar and Bacakground images from the web\r\n",
        "#@markdown 1. You can change the URLs to your **own** stuff!\r\n",
        "#@markdown 2. Alternatively, you can upload **local** files in the next cells\r\n",
        " \r\n",
        "video_url = 'https://www.youtube.com/watch?v=NdEPmitJvaA' #@param {type:\"string\"}\r\n",
        "image_url = 'https://i.pinimg.com/564x/74/9e/96/749e96293399ebe1b8c99d89be522383.jpg' #@param {type:\"string\"}\r\n",
        "background_url = 'https://c.pxhere.com/photos/a9/9a/forest_tree_trees_foliage_leaves_leaf_nature_needles-1005931.jpg!d' #@param {type:\"string\"}\r\n",
        "\r\n",
        "if video_url:\r\n",
        "  !rm -f /content/video.mp4\r\n",
        "  !youtube-dl -f 'bestvideo[ext=mp4][vcodec!*=av01][height<=720]+bestaudio[ext=m4a]/mp4[height<=720][vcodec!*=av01]/mp4[vcodec!*=av01]/mp4' '$video_url' --merge-output-format mp4 -o /content/video\r\n",
        "  !mv /content/video.mp4 /content/video \r\n",
        " \r\n",
        "if image_url:\r\n",
        "  !wget '$image_url' -O /content/image\r\n",
        "\r\n",
        "if background_url:\r\n",
        "  !wget '$background_url' -O /content/background"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4L4lDMont6aP",
        "cellView": "form"
      },
      "source": [
        "#@title Optionally upload local Driver video { run: \"auto\" }\r\n",
        "manually_upload_video = False #@param {type:\"boolean\"}\r\n",
        "if manually_upload_video:\r\n",
        "  from google.colab import files\r\n",
        "  import shutil\r\n",
        "\r\n",
        "  %cd /content/sample_data\r\n",
        "  try:\r\n",
        "    uploaded = files.upload()\r\n",
        "  except Exception as e:\r\n",
        "    %cd /content\r\n",
        "    raise e\r\n",
        "\r\n",
        "  for fn in uploaded:\r\n",
        "    shutil.move('/content/sample_data/'+fn, '/content/video')\r\n",
        "    break\r\n",
        "  %cd /content"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ExeY9p-Ht-_6",
        "cellView": "form"
      },
      "source": [
        "#@title Optionally upload local Avatar image { run: \"auto\" }\r\n",
        "manually_upload_image = False #@param {type:\"boolean\"}\r\n",
        "if manually_upload_image:\r\n",
        "  from google.colab import files\r\n",
        "  import shutil\r\n",
        " \r\n",
        "  %cd /content/sample_data\r\n",
        "  try:\r\n",
        "    uploaded = files.upload()\r\n",
        "  except Exception as e:\r\n",
        "    %cd /content\r\n",
        "    raise e\r\n",
        "\r\n",
        "  for fn in uploaded:\r\n",
        "    shutil.move('/content/sample_data/'+fn, '/content/image')\r\n",
        "    break\r\n",
        "  %cd /content"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "7b6UW1KM2Myw"
      },
      "source": [
        "#@title Optionally upload local Background image { run: \"auto\" }\r\n",
        "manually_upload_background = False #@param {type:\"boolean\"}\r\n",
        "if manually_upload_background:\r\n",
        "  from google.colab import files\r\n",
        "  import shutil\r\n",
        " \r\n",
        "  %cd /content/sample_data\r\n",
        "  try:\r\n",
        "    uploaded = files.upload()\r\n",
        "  except Exception as e:\r\n",
        "    %cd /content\r\n",
        "    raise e\r\n",
        "\r\n",
        "  for fn in uploaded:\r\n",
        "    shutil.move('/content/sample_data/'+fn, '/content/background')\r\n",
        "    break\r\n",
        "  %cd /content"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eUBfTA8YuEAy",
        "cellView": "form"
      },
      "source": [
        "#@title Optionally shorten Driver video\r\n",
        "start_seconds = 0 #@param {type:\"number\"}\r\n",
        "duration_seconds =  60#@param {type:\"number\"}\r\n",
        "start_seconds = max(start_seconds,0)\r\n",
        "duration_seconds = max(duration_seconds,0)\r\n",
        "\r\n",
        "if duration_seconds: \r\n",
        "  !mv /content/video /content/full_video\r\n",
        "  !ffmpeg -ss $start_seconds -t $duration_seconds -i /content/full_video -f mp4 /content/video -y"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cQ7R6KGHuEqI",
        "cellView": "form"
      },
      "source": [
        "#@title Prepare assets\r\n",
        "#@markdown If you ran out of RAM this means that the video is too large. You can shorten it above.\r\n",
        " \r\n",
        "#center_video_to_body = True #@param {type:\"boolean\"}\r\n",
        "#crop_video_to_body = True #@param {type:\"boolean\"}\r\n",
        "#video_crop_expansion_factor = 1.05 #@param {type:\"number\"}\r\n",
        "center_image_to_body = True #@param {type:\"boolean\"}\r\n",
        "crop_image_to_body = False #@param {type:\"boolean\"}\r\n",
        "image_crop_expansion_factor = 1.05 #@param {type:\"number\"}\r\n",
        "#video_crop_expansion_factor = max(video_crop_expansion_factor, 1)\r\n",
        "image_crop_expansion_factor = max(image_crop_expansion_factor, 1)\r\n",
        "keep_aspect_background = True #@param {type:\"boolean\"}\r\n",
        "image_size = 512\r\n",
        "\r\n",
        "import imageio\r\n",
        "import numpy as np\r\n",
        "import matplotlib.pyplot as plt\r\n",
        "import matplotlib.animation as animation\r\n",
        "from skimage.transform import resize\r\n",
        "from IPython.display import HTML, clear_output\r\n",
        "import cv2\r\n",
        "import shutil\r\n",
        "import os\r\n",
        "import warnings\r\n",
        "warnings.filterwarnings(\"ignore\")\r\n",
        " \r\n",
        "hog = cv2.HOGDescriptor()\r\n",
        "hog.setSVMDetector(cv2.HOGDescriptor_getDefaultPeopleDetector())\r\n",
        " \r\n",
        "def fix_dims(im):\r\n",
        "    if im.ndim == 2:\r\n",
        "        im = np.tile(im[..., None], [1, 1, 3])\r\n",
        "    return im[...,:3]\r\n",
        " \r\n",
        "def get_crop(im, center_body=True, crop_body=True, expansion_factor=1, rects=None):\r\n",
        "    im = fix_dims(im)\r\n",
        "    if (center_body or crop_body) and rects is None:\r\n",
        "        rects, _ = hog.detectMultiScale(im, winStride=(4, 4),padding=(8,8), scale=expansion_factor)\r\n",
        "    if (center_body or crop_body) and rects is not None and len(rects):\r\n",
        "        x0,y0,w,h = sorted(rects, key=lambda x: x[2]*x[3])[-1]\r\n",
        "        if crop_body:\r\n",
        "            x0 += w//2-h//2\r\n",
        "            x1 = x0+h\r\n",
        "            y1 = y0+h\r\n",
        "        else:\r\n",
        "            img_h,img_w = im.shape[:2]\r\n",
        "            x0 += (w-img_h)//2\r\n",
        "            x1 = x0+img_h\r\n",
        "            y0 = 0\r\n",
        "            y1 = img_h\r\n",
        "    else:\r\n",
        "        h,w = im.shape[:2]\r\n",
        "        x0 = (w-h)//2\r\n",
        "        x1 = (w+h)//2\r\n",
        "        y0 = 0\r\n",
        "        y1 = h\r\n",
        "    return int(x0),int(x1),int(y0),int(y1)\r\n",
        " \r\n",
        "def pad_crop_resize(im, x0=None, x1=None, y0=None, y1=None, new_h=image_size, new_w=image_size):\r\n",
        "    im = fix_dims(im)\r\n",
        "    h,w = im.shape[:2]\r\n",
        "    if x0 is None:\r\n",
        "      x0 = 0\r\n",
        "    if x1 is None:\r\n",
        "      x1 = w\r\n",
        "    if y0 is None:\r\n",
        "      y0 = 0\r\n",
        "    if y1 is None:\r\n",
        "      y1 = h\r\n",
        "    if x0<0 or x1>w or y0<0 or y1>h:\r\n",
        "        im = np.pad(im, pad_width=[(max(-y0,0),max(y1-h,0)),(max(-x0,0),max(x1-w,0)),(0,0)], mode='edge')\r\n",
        "    return resize(im[max(y0,0):y1-min(y0,0),max(x0,0):x1-min(x0,0)], (new_h, new_w))\r\n",
        "\r\n",
        "def crop_resize(im, size, crop=False):\r\n",
        "  if im.shape[:2] == size:\r\n",
        "    return im\r\n",
        "  if size[0]<im.shape[0] or size[1]<im.shape[1]:\r\n",
        "    interp = cv2.INTER_AREA\r\n",
        "  else:\r\n",
        "    interp = cv2.INTER_CUBIC\r\n",
        "  if not crop:\r\n",
        "    return np.clip(cv2.resize(im, size[::-1], interpolation=interp),0,1)\r\n",
        "  ratio = max(size[0]/im.shape[0], size[1]/im.shape[1])\r\n",
        "  im = np.clip(cv2.resize(im, (int(np.ceil(im.shape[1]*ratio)), int(np.ceil(im.shape[0]*ratio))), interpolation=interp),0,1)\r\n",
        "  return im[(im.shape[0]-size[0])//2:(im.shape[0]-size[0])//2+size[0], (im.shape[1]-size[1])//2:(im.shape[1]-size[1])//2+size[1]]\r\n",
        " \r\n",
        "source_image = imageio.imread('/content/image')\r\n",
        "source_image = pad_crop_resize(source_image, *get_crop(source_image, center_body=center_image_to_body, crop_body=crop_image_to_body, expansion_factor=image_crop_expansion_factor))\r\n",
        "imageio.imwrite('/content/crop.png', (source_image*255).astype(np.uint8))\r\n",
        "\r\n",
        "if os.path.exists('/content/background'):\r\n",
        "  bg_image = imageio.imread('/content/background')\r\n",
        "  bg_image = crop_resize(bg_image/255, source_image.shape[:2], crop=keep_aspect_background)\r\n",
        "  imageio.imwrite('/content/bg_crop.png', (bg_image*255).astype(np.uint8))\r\n",
        "\r\n",
        "#shutil.rmtree('/content/images', ignore_errors=True)\r\n",
        "#os.makedirs('/content/images')\r\n",
        "with imageio.get_reader('/content/video', format='mp4') as reader:\r\n",
        "  fps = reader.get_meta_data()['fps']\r\n",
        "''' \r\n",
        "  driving_video = []\r\n",
        "  rects = None\r\n",
        "  try:\r\n",
        "      for i,im in enumerate(reader):\r\n",
        "          if not crop_video_to_body:\r\n",
        "              break\r\n",
        "          rects, _ = hog.detectMultiScale(im, winStride=(4, 4),padding=(8,8), scale=video_crop_expansion_factor)\r\n",
        "          if rects is not None and len(rects):\r\n",
        "              break\r\n",
        "      x0,x1,y0,y1 = get_crop(im, center_body=center_video_to_body, crop_body=crop_video_to_body, expansion_factor=video_crop_expansion_factor, rects=rects)\r\n",
        "      reader.set_image_index(0)\r\n",
        "      for j,im in enumerate(reader):\r\n",
        "          vid_frame = pad_crop_resize(im,x0,x1,y0,y1)\r\n",
        "          #driving_video.append(vid_frame)\r\n",
        "          imageio.imwrite(os.path.join('/content/images','%05d.jpg'%j), (vid_frame*255).astype(np.uint8))\r\n",
        "  except RuntimeError:\r\n",
        "      pass\r\n",
        " \r\n",
        "def vid_display(source, driving, generated=None):\r\n",
        "    fig = plt.figure(figsize=(8 + 4 * (generated is not None), 6))\r\n",
        " \r\n",
        "    ims = []\r\n",
        "    for i in range(len(driving)):\r\n",
        "        cols = [source]\r\n",
        "        cols.append(driving[i])\r\n",
        "        if generated is not None:\r\n",
        "            cols.append(generated[i])\r\n",
        "        im = plt.imshow(np.concatenate(cols, axis=1), animated=True)\r\n",
        "        plt.axis('off')\r\n",
        "        ims.append([im])\r\n",
        " \r\n",
        "    ani = animation.ArtistAnimation(fig, ims, interval=50, repeat_delay=1000)\r\n",
        "    plt.close()\r\n",
        "    return ani\r\n",
        " \r\n",
        "clear_output()\r\n",
        "if rects is not None and len(rects):\r\n",
        "    print('first found body in frame %d'%i)\r\n",
        "print('number of frames: %d'%j)\r\n",
        "HTML(vid_display(source_image, driving_video).to_html5_video())\r\n",
        "'''\r\n",
        "pass"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BJskZDTiEKMB",
        "cellView": "form"
      },
      "source": [
        "#@title Animate\n",
        "#@markdown 1. The 'pose_fc' is the smooth factor of the temporal poses. The smaller of this value, the smoother of the temporal poses.\n",
        "#@markdown 2. The 'cam_fc' is the smooth factor of the temporal cameras (locations in the image space). The smaller of this value, the smoother of the locations in sequences.\n",
        "\n",
        "pose_fc = 300 #@param {type:'integer'}\n",
        "cam_fc = 100 #@param {type:'integer'}\n",
        "background = 'replace' #@param ['none', 'inpaint', 'replace']\n",
        "\n",
        "%cd /content/iPERCore\n",
        "\n",
        "import os.path as osp\n",
        "import platform\n",
        "import argparse\n",
        "import time\n",
        "import sys\n",
        "import subprocess\n",
        "from IPython.display import HTML\n",
        "from base64 import b64encode\n",
        "\n",
        "# the gpu ids\n",
        "gpu_ids = \"0\"\n",
        "\n",
        "# the default number of source images, it will be updated if the actual number of sources <= num_source\n",
        "num_source = 1\n",
        "\n",
        "# the assets directory. This is very important, please download it from `one_drive_url` firstly.\n",
        "assets_dir = \"/content/iPERCore/assets\"\n",
        "\n",
        "# the output directory.\n",
        "output_dir = \"./results\"\n",
        "shutil.rmtree(output_dir, ignore_errors=True)\n",
        "\n",
        "# symlink from the actual assets directory to this current directory\n",
        "work_asserts_dir = os.path.join(\"./assets\")\n",
        "if not os.path.exists(work_asserts_dir):\n",
        "    os.symlink(osp.abspath(assets_dir), osp.abspath(work_asserts_dir),\n",
        "               target_is_directory=(platform.system() == \"Windows\"))\n",
        "\n",
        "cfg_path = osp.join(work_asserts_dir, \"configs\", \"deploy.toml\")\n",
        "\n",
        "# This is a specific model name, and it will be used if you do not change it. This is the case of `trump`\n",
        "model_id = \"mymodel\"\n",
        "\n",
        "# the source input information, here \\\" is escape character of double duote \"\n",
        "src_path = \"\\\"path?=/content/crop.png,name?=mymodel\"\n",
        "if background=='replace' and os.path.exists('/content/bg_crop.png'):\n",
        "  src_path += ',bg_path?=/content/bg_crop.png'\n",
        "src_path += '\"'\n",
        "\n",
        "!cp /content/video /content/video.mp4\n",
        "ref_path = \"\\\"path?=/content/video.mp4,\"  \\\n",
        "             \"name?=myoutput,\" \\\n",
        "             \"pose_fc?=%d,\"\\\n",
        "             \"cam_fc?=%d,\"\\\n",
        "             \"fps?=%f\\\"\"%(pose_fc,cam_fc,fps)\n",
        "options = ''\n",
        "if background=='inpaint':\n",
        "  options += ' --use_inpaintor'\n",
        "!python -m iPERCore.services.run_imitator --gpu_ids $gpu_ids --num_source $num_source --image_size $image_size --output_dir $output_dir --model_id $model_id --cfg_path $cfg_path --src_path $src_path --ref_path $ref_path $options\n",
        "\n",
        "clear_output()\n",
        "mp4 = open(\"./results/primitives/mymodel/synthesis/imitations/mymodel-myoutput.mp4\", \"rb\").read()\n",
        "data_url = \"data:video/mp4;base64,\" + b64encode(mp4).decode()\n",
        "HTML(f\"\"\"\n",
        "<video width=\"100%\" height=\"100%\" controls autoplay loop>\n",
        "      <source src=\"{data_url}\" type=\"video/mp4\">\n",
        "</video>\"\"\")\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "NCU7PkLNrLOm"
      },
      "source": [
        "#@title Download\r\n",
        "#@markdown 1. If it fails try running this cell again.\r\n",
        "#@markdown 2. Alternatively, you can manually download \"output.mp4\", \"combined.mp4\" from the folder on the left (click \"Refresh\" if missing).\r\n",
        "\r\n",
        "print() #see https://github.com/googlecolab/colabtools/issues/468\r\n",
        "from google.colab import files\r\n",
        "files.download('./results/primitives/mymodel/synthesis/imitations/mymodel-myoutput.mp4') #fails for Firefox private window"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}