{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "together-subsection",
   "metadata": {},
   "source": [
    "# Image Duplicates & Near Duplicates\n",
    "\n",
    "This example shows how SentenceTransformers can be used to find image duplicates and near duplicats.\n",
    "\n",
    "As model, we use the [OpenAI CLIP Model](https://github.com/openai/CLIP), which was trained on a large set of images and image alt texts.\n",
    "\n",
    "As a source for fotos, we use the [Unsplash Dataset Lite](https://unsplash.com/data), which contains about 25k images. See the [License](https://unsplash.com/license) about the Unsplash images. \n",
    "\n",
    "We encode all images into vector space and then find high density regions in this vector space, i.e., regions where the images are fairly similar."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "mature-separation",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer, util\n",
    "from PIL import Image\n",
    "import glob\n",
    "import torch\n",
    "import pickle\n",
    "import zipfile\n",
    "from IPython.display import display\n",
    "from IPython.display import Image as IPImage\n",
    "import os\n",
    "from tqdm.autonotebook import tqdm\n",
    "\n",
    "#First, we load the CLIP model\n",
    "model = SentenceTransformer('clip-ViT-B-32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "disciplinary-freight",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Next, we get about 25k images from Unsplash \n",
    "img_folder = 'photos/'\n",
    "if not os.path.exists(img_folder) or len(os.listdir(img_folder)) == 0:\n",
    "    os.makedirs(img_folder, exist_ok=True)\n",
    "    \n",
    "    photo_filename = 'unsplash-25k-photos.zip'\n",
    "    if not os.path.exists(photo_filename):   #Download dataset if does not exist\n",
    "        util.http_get('http://sbert.net/datasets/'+photo_filename, photo_filename)\n",
    "        \n",
    "    #Extract all images\n",
    "    with zipfile.ZipFile(photo_filename, 'r') as zf:\n",
    "        for member in tqdm(zf.infolist(), desc='Extracting'):\n",
    "            zf.extract(member, img_folder)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "running-grammar",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Images: 24996\n"
     ]
    }
   ],
   "source": [
    "# Now, we need to compute the embeddings\n",
    "# To speed things up, we destribute pre-computed embeddings\n",
    "# Otherwise you can also encode the images yourself.\n",
    "# To encode an image, you can use the following code:\n",
    "# from PIL import Image\n",
    "# img_emb = model.encode(Image.open(filepath))\n",
    "\n",
    "use_precomputed_embeddings = True\n",
    "\n",
    "if use_precomputed_embeddings: \n",
    "    emb_filename = 'unsplash-25k-photos-embeddings.pkl'\n",
    "    if not os.path.exists(emb_filename):   #Download dataset if does not exist\n",
    "        util.http_get('http://sbert.net/datasets/'+emb_filename, emb_filename)\n",
    "        \n",
    "    with open(emb_filename, 'rb') as fIn:\n",
    "        img_names, img_emb = pickle.load(fIn)  \n",
    "    print(\"Images:\", len(img_names))\n",
    "else:\n",
    "    img_names = list(glob.glob('photos/*.jpg'))\n",
    "    print(\"Images:\", len(img_names))\n",
    "    img_emb = model.encode([Image.open(filepath) for filepath in img_names], batch_size=128, convert_to_tensor=True, show_progress_bar=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dutch-wallet",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now we run the clustering algorithm\n",
    "# With the threshold parameter, we define at which threshold we identify\n",
    "# two images as similar. Set the threshold lower, and you will get larger clusters which have \n",
    "# less similar images in it (e.g. black cat images vs. cat images vs. animal images).\n",
    "# With min_community_size, we define that we only want to have clusters of a certain minimal size\n",
    "\n",
    "duplicates = util.paraphrase_mining_embeddings(img_emb)\n",
    "\n",
    "# duplicates contains a list with triplets (score, image_id1, image_id2) and is scorted in decreasing order"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "middle-cable",
   "metadata": {},
   "source": [
    "## Duplicates\n",
    "In the next cell, we output the top 10 most similar images. These are identical images,\n",
    "i.e. the same photo was uploaded twice to Unsplash"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "adjacent-cutting",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Score: 1.000\n",
      "0UtMDLOk0Vg.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 83496 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10OY7Od4YeQ.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 83496 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Score: 1.000\n",
      "QO4Y97jiVDQ.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 27600 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0ETSZYPjvDo.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 27600 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Score: 1.000\n",
      "fPipNm8A5uc.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 154348 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cSSWEsnYv7w.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 154348 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Score: 1.000\n",
      "xm6esles7ds.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 77420 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CkwM5Mrnd8U.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 77420 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Score: 1.000\n",
      "4f4e3hRnwKs.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 107436 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f3hDGOHptrM.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 107436 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Score: 1.000\n",
      "LYycu62H2-8.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 216252 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "s5_rlPwwads.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 216252 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Score: 1.000\n",
      "RadJczno8RA.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 40728 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fmTDdLOEkl0.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 40728 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Score: 1.000\n",
      "6VBgzF3H64c.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 100356 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mSL1shtsWFU.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 100356 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Score: 1.000\n",
      "PySq7yX6hlw.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 20700 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "JG_HfydoNqY.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 20700 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Score: 1.000\n",
      "Aq3NQwdqOU8.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 245600 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "JO_6maFFeoQ.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 245600 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for score, idx1, idx2 in duplicates[0:10]:\n",
    "    print(\"\\nScore: {:.3f}\".format(score))\n",
    "    print(img_names[idx1])\n",
    "    display(IPImage(os.path.join(img_folder, img_names[idx1]), width=200))\n",
    "    print( img_names[idx2])\n",
    "    display(IPImage(os.path.join(img_folder, img_names[idx2]), width=200))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "static-member",
   "metadata": {},
   "source": [
    "## Near Duplicates\n",
    "We can also skip the duplicate images and find near duplicates.\n",
    "To achieve this, we only look at images pairs that have a cosine similarity below a certain threshold.\n",
    "In our example, we look at images with a cosine similarity lower than 0.99"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "nearby-kentucky",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Score: 0.989\n",
      "ht07CBODJVY.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 64912 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TVpMXc3Urzg.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 58208 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Score: 0.989\n",
      "mOLet_-xn2M.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 140852 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-zEDq4sRxRE.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 197932 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Score: 0.989\n",
      "474zbfqYDIA.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 94056 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dSB_QwlCpRs.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 103772 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Score: 0.988\n",
      "2yqsIIFSN-Y.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 45692 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qcmhNEipNl0.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 40728 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Score: 0.988\n",
      "xjM0j6qBPGM.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 111896 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "eS7HrvG0mcA.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 113424 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Score: 0.988\n",
      "IXO5G1jR4h4.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 49504 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EheKbIZ8oAw.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 56472 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Score: 0.988\n",
      "SKf6puU2BV8.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 54872 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IXO5G1jR4h4.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 49504 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Score: 0.987\n",
      "LdhDmizaN7k.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 174512 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1VO4qqSvxfA.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 100140 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Score: 0.987\n",
      "wH3POmZAsio.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 64324 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8jeWeKdygfk.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 124068 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Score: 0.986\n",
      "up_IHPsJvSI.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 46052 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "zlCAI1SWZ5M.jpg\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 61560 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/jpeg": {
       "width": 200
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "threshold = 0.99\n",
    "near_duplicates = [entry for entry in duplicates if entry[0] < threshold]\n",
    "\n",
    "for score, idx1, idx2 in near_duplicates[0:10]:\n",
    "    print(\"\\nScore: {:.3f}\".format(score))\n",
    "    print(img_names[idx1])\n",
    "    display(IPImage(os.path.join(img_folder, img_names[idx1]), width=200))\n",
    "    print(img_names[idx2])\n",
    "    display(IPImage(os.path.join(img_folder, img_names[idx2]), width=200))\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
