facevidblur.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "T4",
      "cell_execution_strategy": "setup",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/eyaler/avatars4all/blob/master/facevidblur.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#FaceVidBlur\n",
        "\n",
        "### Notebook by [Eyal Gruss](https://eyalgruss.com), [@eyaler](twitter.com/eyaler)\n",
        "\n",
        "Blur (some) faces in video + optionally draw facial landmarks\n",
        "\n",
        "Notes:\n",
        "1. You should always manually inspect the output video to make sure that the faces you want to blur do not show in some frames due to miss detections.\n",
        "2. For the use case of bluring *only some* of the faces, the current solution depends on having a consistent spatial ordering of faces. This may break as people move too much, get in/out of the frame, get their face occluded or turn around, as well as other face detection misses and false positives. For some of these cases the default safe_mode will try to blur extra faces to be on the safe side. I am currently working on a more robust solution.\n",
        "\n",
        "More notebooks: [github.com/eyaler/avatars4all](https://github.com/eyaler/avatars4all)\n",
        "\n",
        "Shortcut here: [tfi.la/blur](https://tfi.la/blur)\n",
        "\n",
        "Something not working? Open an [issue](https://github.com/eyaler/avatars4all/issues)\n",
        "\n",
        "If you find my work useful please consider supporting me via [GitHub Sponsors](https://github.com/sponsors/eyaler) or [PayPal](https://www.paypal.com/donate/?hosted_button_id=LNJ6F3FR79ARE)"
      ],
      "metadata": {
        "id": "fIHL9q7CJ4tR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Setup\n",
        "\n",
        "import locale\n",
        "locale.getpreferredencoding = lambda: 'UTF-8'\n",
        "\n",
        "!pip install git+https://github.com/ytdl-org/youtube-dl\n",
        "!pip install git+https://github.com/1adrianb/face-alignment"
      ],
      "metadata": {
        "cellView": "form",
        "id": "NrVPvyVku1BJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Optionally mount Google Drive (MARK CHECKBOX) { run: \"auto\" }\n",
        "mount_google_drive = False #@param {type:\"boolean\"}\n",
        "if mount_google_drive:\n",
        "  from google.colab import drive\n",
        "  drive.mount('/content/drive')\n",
        "  print('path is /content/drive/MyDrive')"
      ],
      "metadata": {
        "cellView": "form",
        "id": "yUbiJMGMczmS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ANINpF9JKuk8",
        "cellView": "form"
      },
      "source": [
        "#@title Blur faces in video\n",
        "\n",
        "video_url = 'https://www.youtube.com/watch?v=SoAKSHcrDGg' #@param {type: 'string'}\n",
        "#@markdown (leave empty to upload file - A BUTTON WILL APPEAR BELOW, or link to youtube / vimeo / video url / path to video on mounted drive [drive/MyDrive/...] / path to video on colab)\n",
        "i_just_uploaded_a_file_and_i_want_to_reuse_that_instead_of_uploading_a_new_one = False #@param {type: 'boolean'}\n",
        "face_order = 'left-to-right then top-to-bottom' #@param ['left-to-right then top-to-bottom', 'right-to-left then top-to-bottom', 'top-to-bottom then left-to-right', 'bottom-to-top then left-to-right']\n",
        "#@markdown (one of these orderings may be more stable for your video, as order is recalculated per frame)\n",
        "face_num = '' #@param {type: 'string'}\n",
        "#@markdown (leave empty to blur all faces, or use face_num = 1,2,... to blur one or more faces from all the faces ordered as above, use face_num = -1,-2,... to blur all faces except these)\n",
        "safe_mode = True #@param {type: 'boolean'}\n",
        "#@markdown (hide neighbour faces when the number of faces changes)\n",
        "discard_faces_with_longest_dimension_pixels_below = 0 #@param {type: 'integer'}\n",
        "min_conf = 0.9 #@param {type: 'number'}\n",
        "#@markdown (lower min_conf if faces are not detected, raise min_conf if there are false detections that you cannot get rid of by using discard_faces_with_longest_dimension_pixels_below)\n",
        "radius_factor = 1 #@param {type: 'number'}\n",
        "pixelization_areas_per_dimension = 1 #@param {type: 'integer'}\n",
        "pixelization_colors_per_area = '1' #@param ['1', '2']\n",
        "expose_eyes_mouth = False #@param {type: 'boolean'}\n",
        "draw_landmarks = True #@param {type: 'boolean'}\n",
        "#@markdown (uncheck this if you don't want the stupid lines over the faces)\n",
        "landmarks_hex = '0000FF' #@param {type: 'string'}\n",
        "start_seconds = 0 #@param {type: 'number'}\n",
        "duration_seconds = 15 #@param {type: 'number'}\n",
        "#@markdown (use duration_seconds = 0 for unrestricted duration)\n",
        "max_width = 0 #@param {type: 'integer'}\n",
        "#@markdown (use max_width = 0 for unrestricted width)\n",
        "max_height = 0 #@param {type: 'integer'}\n",
        "#@markdown (use max_height = 0 for unrestricted height)\n",
        "copy_audio = True  #@param {type: 'boolean'}\n",
        "output_filepath = 'output.mp4' #@param {type: 'string'}\n",
        "#@markdown (you can specify a path to save on mounted drive [drive/MyDrive/...])\n",
        "\n",
        "start_seconds = start_seconds or 0\n",
        "duration_seconds = duration_seconds or 0\n",
        "max_width = max_width or 0\n",
        "max_height = max_height or 0\n",
        "\n",
        "from time import time\n",
        "start_time = time()\n",
        "\n",
        "%cd /content\n",
        "\n",
        "import os\n",
        "from google.colab import files\n",
        "output_filepath = os.path.normpath(output_filepath.replace('\\\\', '/'))\n",
        "output_path, output_filename = os.path.split(output_filepath)\n",
        "\n",
        "need_dl = True\n",
        "try:\n",
        "  if orig_video and video_url == save_url and (video_url or i_just_uploaded_a_file_and_i_want_to_reuse_that_instead_of_uploading_a_new_one):\n",
        "    need_dl = False\n",
        "except:\n",
        "  pass\n",
        "if need_dl:\n",
        "  if not video_url:\n",
        "    %cd /content/sample_data\n",
        "    try:\n",
        "      uploaded = files.upload()\n",
        "    except Exception:\n",
        "      %cd /content\n",
        "      raise\n",
        "    for fn in uploaded:\n",
        "      orig_video = os.path.abspath(fn)\n",
        "      break\n",
        "    else:\n",
        "      need_dl = False\n",
        "    %cd /content\n",
        "  elif os.path.isfile(os.path.normpath(video_url.replace('\\\\', '/'))):\n",
        "    orig_video = os.path.abspath(os.path.normpath(video_url.replace('\\\\', '/')))\n",
        "  elif os.path.isfile(os.path.normpath('/content/drive/MyDrive/' + video_url.replace('\\\\', '/'))):\n",
        "    orig_video = os.path.normpath('/content/drive/MyDrive/' + video_url.replace('\\\\', '/'))\n",
        "  else:\n",
        "    orig_video = '/content/orig_video.mp4'\n",
        "    !rm -f $orig_video\n",
        "    !youtube-dl --no-playlist -f \"bestvideo[ext=mp4][vcodec!*=av01]+bestaudio[ext=m4a]/mp4[vcodec!*=av01]/bestvideo[ext=mp4]+bestaudio[ext=m4a]/bestvideo[ext=mp4]+bestaudio/mp4\" \"$video_url\" --merge-output-format mp4 -o $orig_video\n",
        "    if not os.path.exists(orig_video):\n",
        "      orig_video = '/content/orig_video.' + video_url.rsplit('.', 1)[1]\n",
        "      !wget \"$video_url\" -O $orig_video\n",
        "      if not os.path.getsize(orig_video):\n",
        "        !rm -f $orig_video\n",
        "    if not os.path.exists(orig_video):\n",
        "      orig_video = None\n",
        "      raise FileNotFoundError\n",
        "  input_video = '/content/input_video' + os.path.splitext(orig_video)[-1]\n",
        "\n",
        "need_fix_duration = True\n",
        "try:\n",
        "  if not need_dl and start_seconds == save_start_seconds and duration_seconds == save_duration_seconds:\n",
        "    need_fix_duration = False\n",
        "except:\n",
        "  pass\n",
        "if need_fix_duration:\n",
        "  if start_seconds or duration_seconds:\n",
        "    !ffmpeg -y -ss $start_seconds -t $duration_seconds -i \"$orig_video\" -f mp4 $input_video\n",
        "  else:\n",
        "    !cp \"$orig_video\" $input_video\n",
        "\n",
        "save_url = video_url\n",
        "save_start_seconds = start_seconds\n",
        "save_duration_seconds = duration_seconds\n",
        "\n",
        "import cv2\n",
        "import face_alignment\n",
        "face_detector_kwargs = {}\n",
        "if min_conf < 0.5:\n",
        "  face_detector_kwargs = {'filter_threshold': min_conf}\n",
        "fa = face_alignment.FaceAlignment(landmarks_type=1, face_detector_kwargs=face_detector_kwargs)\n",
        "import imageio.v3 as iio\n",
        "from IPython.display import display, Image, Video, clear_output\n",
        "import numpy as np\n",
        "from skimage.draw import ellipse, polygon, polygon_perimeter\n",
        "\n",
        "!rm -f /content/numbers.png\n",
        "!rm -rf /content/in_frames\n",
        "!mkdir -p /content/in_frames\n",
        "\n",
        "face_parts = {'eyebrow1': slice(17, 22),\n",
        "              'eyebrow2': slice(22, 27),\n",
        "              'nose': slice(27, 31),\n",
        "              'nostril': slice(31, 36),\n",
        "              'eye1': slice(36, 42),\n",
        "              'eye2': slice(42, 48),\n",
        "              'lips': slice(48, 60),\n",
        "              'teeth': slice(60, 68)\n",
        "              }\n",
        "step = len(landmarks_hex) // 3\n",
        "landmarks_rgb = [int(landmarks_hex[i: i + step], 16) * 17**(step == 1) for i in range(0, len(landmarks_hex), step)]\n",
        "\n",
        "face_num = face_num or None\n",
        "if face_num is not None:\n",
        "  face_num = [int(i) for i in face_num.split(',')]\n",
        "  assert all(i > 0 for i in face_num) or all(i < 0 for i in face_num), 'face_num should be either empty or all positives or all negatives'\n",
        "  neg_mode = face_num[0] < 0\n",
        "  face_num = [abs(i) - 1 for i in face_num]\n",
        "\n",
        "first_faces = ''\n",
        "max_face_len_unfiltered = 0\n",
        "max_face_len_below = 0\n",
        "min_face_len_above = None\n",
        "max_face_conf_unfiltered = 0\n",
        "max_face_conf_below = 0\n",
        "min_face_conf_above = 1\n",
        "max_filtered_faces_in_frame = 0\n",
        "frame_of_max_filtered_faces_in_frame = 0\n",
        "for frame, im in enumerate(iio.imiter(input_video)):\n",
        "  if min_face_len_above is None:\n",
        "    min_face_len_above = max(im.shape[:2])\n",
        "  landmarks, _, bboxes = fa.get_landmarks_from_image(im, return_bboxes=True)\n",
        "  if bboxes is not None:\n",
        "    max_face_len_unfiltered = max(max_face_len_unfiltered, *[max(x1 - x0, y1 - y0) + 1 for x0, y0, x1, y1, _ in bboxes])\n",
        "    max_face_conf_unfiltered = max(max_face_conf_unfiltered, *[b[-1] for b in bboxes])\n",
        "    lb = list(zip(landmarks, bboxes))\n",
        "\n",
        "    if discard_faces_with_longest_dimension_pixels_below:\n",
        "      for x0, y0, x1, y1, _ in bboxes:\n",
        "        m = max(x1 - x0, y1 - y0) + 1\n",
        "        if m < discard_faces_with_longest_dimension_pixels_below:\n",
        "          max_face_len_below = max(max_face_len_below, m)\n",
        "      lb = [(l, b) for l, b in lb if max(b[2] - b[0], b[3] - b[1]) + 1 >= discard_faces_with_longest_dimension_pixels_below]\n",
        "    if lb:\n",
        "      min_face_len_above = min(min_face_len_above, *[max(x1 - x0, y1 - y0) + 1 for _, (x0, y0, x1, y1, _) in lb])\n",
        "\n",
        "    if min_conf:\n",
        "      for l, b in lb:\n",
        "        if b[-1] < min_conf:\n",
        "          max_face_conf_below = max(max_face_conf_below, b[-1])\n",
        "      lb = [(l, b) for l, b in lb if b[-1] >= min_conf]\n",
        "    if lb:\n",
        "      min_face_conf_above = min(min_face_conf_above, *[b[-1] for l, b in lb])\n",
        "\n",
        "    lb = sorted(lb, key=lambda b: (b[1][0], b[1][2], b[1][1], b[1][3]) if face_order == 'left-to-right then top-to-bottom' else (-b[1][2], -b[1][0], b[1][1], b[1][3]) if face_order == 'right-to-left then top-to-bottom' else (b[1][1], b[1][3], b[1][0], b[1][2]) if face_order == 'top-to-bottom then left-to-right' else (-b[1][3], -b[1][1], b[1][0], b[1][2]))\n",
        "    if len(lb) > max_filtered_faces_in_frame:\n",
        "      max_filtered_faces_in_frame = len(lb)\n",
        "      frame_of_max_filtered_faces_in_frame = frame\n",
        "    if len(lb) > 1 and not first_faces:\n",
        "      first_faces = f'{frame=}'\n",
        "      face_count = len(lb)\n",
        "      numbers = im.copy()\n",
        "      for i, (_, (x0, y0, x1, y1, conf)) in enumerate(lb):\n",
        "        cv2.putText(numbers, str(i + 1), (max(min(int(x1), im.shape[1] - 1) - 20, 0), max(min(int(y1), im.shape[0] - 1), 20)), 0, .7, (0, 255, 0), 2)\n",
        "        first_faces += f'; {i + 1}: w={x1 - x0 + 1 :.1f} h={y1 - y0 + 1 :.1f} {conf=:.2f}'\n",
        "      iio.imwrite('/content/numbers.png', numbers, compress_level=1)\n",
        "      clear_output()\n",
        "      display(Image('/content/numbers.png'))\n",
        "      print(first_faces)\n",
        "    if face_num is not None:\n",
        "      diff = len(lb) - face_count\n",
        "      if neg_mode:\n",
        "        faces_to_hide = [i for i in range(len(lb)) if i not in face_num]\n",
        "      else:\n",
        "        faces_to_hide = face_num\n",
        "      if safe_mode and diff:\n",
        "        safe_faces_to_hide = []\n",
        "        for n in faces_to_hide:\n",
        "          start = max(min(n, n + diff), max(safe_faces_to_hide, default=-1) + 1)\n",
        "          end = max(max(n, n + diff), start) + 1\n",
        "          safe_faces_to_hide += list(range(start, end))\n",
        "        faces_to_hide = safe_faces_to_hide\n",
        "      lb = [lb[n] for n in faces_to_hide if len(lb) > n]\n",
        "    orig = im.copy()\n",
        "    for landmarks, (x0, y0, x1, y1, conf) in lb:\n",
        "      x = (x0+x1) / 2\n",
        "      y = (y0+y1) / 2\n",
        "      wr = (x1-x0+1) / 2 * radius_factor\n",
        "      hr = (y1-y0+1) / 2 * radius_factor\n",
        "      yy, xx = ellipse(y, x, hr, wr, im.shape)\n",
        "      ycrcb = cv2.cvtColor(orig[None, yy, xx], cv2.COLOR_RGB2YCrCb)[0]\n",
        "      min_x = min(xx)\n",
        "      spread_x = max(xx) - min_x + 1\n",
        "      min_y = min(yy)\n",
        "      spread_y = max(yy) - min_y + 1\n",
        "\n",
        "      def step(z, minimum, spread):\n",
        "        return (z-minimum) * pixelization_areas_per_dimension // spread % pixelization_areas_per_dimension\n",
        "\n",
        "      areas = [[k for k in range(len(yy)) if step(yy[k], min_y, spread_y) == j and step(xx[k], min_x, spread_x) == i] for j in range(pixelization_areas_per_dimension) for i in range(pixelization_areas_per_dimension)]\n",
        "      for area in areas:\n",
        "        if area:\n",
        "          area = np.array(area)\n",
        "          if pixelization_colors_per_area == '1':\n",
        "            ycrcb[area] = np.median(ycrcb[area], axis=0)\n",
        "          else:\n",
        "            luma = np.argsort(ycrcb[area, 0])\n",
        "            dark = area[luma[: len(luma) // 2]]\n",
        "            bright = area[luma[len(luma) // 2 :]]\n",
        "            ycrcb[dark] = np.median(ycrcb[dark], axis=0)\n",
        "            ycrcb[bright] = np.median(ycrcb[bright], axis=0)\n",
        "      im[yy, xx] = cv2.cvtColor(ycrcb[None, :], cv2.COLOR_YCrCb2RGB)[0]\n",
        "      if expose_eyes_mouth:\n",
        "        for name, part in face_parts.items():\n",
        "          if name in ('eye1', 'eye2', 'lips'):\n",
        "            yy, xx = polygon(*list(zip(*landmarks[part]))[::-1], im.shape)\n",
        "            im[yy, xx] = orig[yy, xx]\n",
        "    if draw_landmarks:\n",
        "      for landmarks, _ in lb:\n",
        "        for part in face_parts.values():\n",
        "          yy, xx = polygon_perimeter(*list(zip(*landmarks[part]))[::-1], im.shape)\n",
        "          im[yy, xx] = landmarks_rgb\n",
        "  iio.imwrite(f'/content/in_frames/frame_{frame:06d}.png', im, compress_level=1)\n",
        "\n",
        "fps = iio.immeta(input_video)['fps']\n",
        "if copy_audio:\n",
        "  !ffmpeg -y -framerate $fps -thread_queue_size 0 -i /content/in_frames/frame_%06d.png -i $input_video -c:v libx264 -c:a aac -map 0:v -map 1:a? -vf \"scale=min(iw\\,$max_width):min(ih\\,$max_height):force_original_aspect_ratio=decrease:force_divisible_by=2\" -pix_fmt yuv420p -profile:v baseline -movflags +faststart \"/content/$output_filename\"\n",
        "else:\n",
        "  !ffmpeg -y -framerate $fps -thread_queue_size 0 -i /content/in_frames/frame_%06d.png -c:v libx264 -vf \"scale=min(iw\\,$max_width):min(ih\\,$max_height):force_original_aspect_ratio=decrease:force_divisible_by=2\" -pix_fmt yuv420p -profile:v baseline -movflags +faststart \"/content/$output_filename\"\n",
        "\n",
        "clear_output()\n",
        "if first_faces:\n",
        "  display(Image('/content/numbers.png'))\n",
        "  print(first_faces)\n",
        "if output_path:\n",
        "  !mkdir -p \"$output_path\"\n",
        "  !cp \"/content/$output_filename\" \"$output_filepath\"\n",
        "meta = iio.immeta('/content/' + output_filename)\n",
        "print(f'took {(time()-start_time) / 60 :.1f} min. orig_video={orig_video}')\n",
        "print(f'output_video=/content/{output_filename} w={meta[\"size\"][0]} h={meta[\"size\"][1]}  aspect_ratio={meta[\"size\"][0] / meta[\"size\"][1] :.3f} t={meta[\"duration\"]} fps={meta[\"fps\"]}')\n",
        "print(f'max_face_len_below_{discard_faces_with_longest_dimension_pixels_below}={max_face_len_below:.1f} min_face_len_above_{discard_faces_with_longest_dimension_pixels_below}={min_face_len_above:.1f} {max_face_len_unfiltered=:.1f}')\n",
        "print(f'max_face_conf_below_{min_conf}={max_face_conf_below:.2f} min_face_conf_above_{min_conf}={min_face_conf_above:.2f} {max_face_conf_unfiltered=:.2f}')\n",
        "print(f'{max_filtered_faces_in_frame=} {frame_of_max_filtered_faces_in_frame=}')\n",
        "print('if video does not show below, you can still download it!')\n",
        "display(Video('/content/' + output_filename, embed=True, html_attributes=\"autoplay controls loop\"))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lH4oJcBOKZcf",
        "cellView": "form"
      },
      "source": [
        "#@title Download\n",
        "\n",
        "from google.colab import files\n",
        "files.download('/content/' + output_filename)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}