{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This tutorial demonstrates how to use int8 operations with OpenCLIP.\n",
    "\n",
    "Usually matrix multiplies are conducted in float16 or bfloat16, but int8 operations are faster.\n",
    "\n",
    "For more information please see https://github.com/mlfoundations/open_clip#int8-support\n",
    "\n",
    "We ran this on an A100 GPU\n",
    "\n",
    "Note that this tutorial requires two additional pip installs on top of those required for standard OpenCLIP."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: triton==2.0.0.post1 in /fsx/home-mitchellw/miniconda3/envs/py2/lib/python3.8/site-packages (2.0.0.post1)\n",
      "Requirement already satisfied: cmake in /fsx/home-mitchellw/miniconda3/envs/py2/lib/python3.8/site-packages (from triton==2.0.0.post1) (3.26.3)\n",
      "Requirement already satisfied: filelock in /fsx/home-mitchellw/miniconda3/envs/py2/lib/python3.8/site-packages (from triton==2.0.0.post1) (3.9.0)\n",
      "Requirement already satisfied: torch in /fsx/home-mitchellw/miniconda3/envs/py2/lib/python3.8/site-packages (from triton==2.0.0.post1) (2.0.0)\n",
      "Requirement already satisfied: lit in /fsx/home-mitchellw/miniconda3/envs/py2/lib/python3.8/site-packages (from triton==2.0.0.post1) (16.0.1)\n",
      "Requirement already satisfied: typing-extensions in /fsx/home-mitchellw/miniconda3/envs/py2/lib/python3.8/site-packages (from torch->triton==2.0.0.post1) (4.4.0)\n",
      "Requirement already satisfied: sympy in /fsx/home-mitchellw/miniconda3/envs/py2/lib/python3.8/site-packages (from torch->triton==2.0.0.post1) (1.11.1)\n",
      "Requirement already satisfied: networkx in /fsx/home-mitchellw/miniconda3/envs/py2/lib/python3.8/site-packages (from torch->triton==2.0.0.post1) (2.8.4)\n",
      "Requirement already satisfied: jinja2 in /fsx/home-mitchellw/miniconda3/envs/py2/lib/python3.8/site-packages (from torch->triton==2.0.0.post1) (3.1.2)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /fsx/home-mitchellw/miniconda3/envs/py2/lib/python3.8/site-packages (from jinja2->torch->triton==2.0.0.post1) (2.1.1)\n",
      "Requirement already satisfied: mpmath>=0.19 in /fsx/home-mitchellw/miniconda3/envs/py2/lib/python3.8/site-packages (from sympy->torch->triton==2.0.0.post1) (1.2.1)\n",
      "Requirement already satisfied: bitsandbytes in /admin/home-mitchellw/git/bitsandbytes (0.38.1)\n",
      "Requirement already satisfied: scikit-image in /fsx/home-mitchellw/miniconda3/envs/py2/lib/python3.8/site-packages (0.20.0)\n",
      "Requirement already satisfied: tifffile>=2019.7.26 in /fsx/home-mitchellw/miniconda3/envs/py2/lib/python3.8/site-packages (from scikit-image) (2023.4.12)\n",
      "Requirement already satisfied: packaging>=20.0 in /fsx/home-mitchellw/miniconda3/envs/py2/lib/python3.8/site-packages (from scikit-image) (23.1)\n",
      "Requirement already satisfied: lazy_loader>=0.1 in /fsx/home-mitchellw/miniconda3/envs/py2/lib/python3.8/site-packages (from scikit-image) (0.2)\n",
      "Requirement already satisfied: scipy<1.9.2,>=1.8 in /fsx/home-mitchellw/miniconda3/envs/py2/lib/python3.8/site-packages (from scikit-image) (1.9.1)\n",
      "Requirement already satisfied: PyWavelets>=1.1.1 in /fsx/home-mitchellw/miniconda3/envs/py2/lib/python3.8/site-packages (from scikit-image) (1.4.1)\n",
      "Requirement already satisfied: pillow>=9.0.1 in /fsx/home-mitchellw/miniconda3/envs/py2/lib/python3.8/site-packages (from scikit-image) (9.4.0)\n",
      "Requirement already satisfied: numpy>=1.21.1 in /fsx/home-mitchellw/miniconda3/envs/py2/lib/python3.8/site-packages (from scikit-image) (1.23.5)\n",
      "Requirement already satisfied: networkx>=2.8 in /fsx/home-mitchellw/miniconda3/envs/py2/lib/python3.8/site-packages (from scikit-image) (2.8.4)\n",
      "Requirement already satisfied: imageio>=2.4.1 in /fsx/home-mitchellw/miniconda3/envs/py2/lib/python3.8/site-packages (from scikit-image) (2.28.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "# necessary installs for int8\n",
    "!pip install triton==2.0.0.post1\n",
    "!pip install bitsandbytes\n",
    "%pip install scikit-image"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's start with using a standard OpenCLIP model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import packages\n",
    "import numpy as np\n",
    "import torch\n",
    "import open_clip\n",
    "from open_clip import tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the model\n",
    "model, _, preprocess = open_clip.create_model_and_transforms('ViT-H-14', pretrained='laion2b_s32b_b79k')\n",
    "model.eval()\n",
    "model = model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 325412 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# lets check out the example image we will be using for classification\n",
    "import matplotlib.pyplot as plt\n",
    "from skimage import data, data_dir\n",
    "import os\n",
    "from PIL import Image\n",
    "%matplotlib inline\n",
    "\n",
    "img = data.astronaut()\n",
    "\n",
    "plt.imshow(img)\n",
    "plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# preprocess image and text\n",
    "img = Image.open(os.path.join(data_dir, 'astronaut.png')).convert(\"RGB\")\n",
    "img_preprocessed = preprocess(img).cuda().unsqueeze(0)\n",
    "\n",
    "descriptions = {\n",
    "    \"page\": \"a page of text about segmentation\",\n",
    "    \"chelsea\": \"a facial photo of a tabby cat\",\n",
    "    \"astronaut\": \"a portrait of an astronaut with the American flag\",\n",
    "    \"rocket\": \"a rocket standing on a launchpad\",\n",
    "    \"motorcycle_right\": \"a red motorcycle standing in a garage\",\n",
    "    \"camera\": \"a person looking at a camera on a tripod\",\n",
    "    \"horse\": \"a black-and-white silhouette of a horse\", \n",
    "    \"coffee\": \"a cup of coffee on a saucer\"\n",
    "}\n",
    "texts = descriptions.values()\n",
    "\n",
    "text_processed = tokenizer.tokenize(texts).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 29236 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# get predictions from the model\n",
    "with torch.cuda.amp.autocast():\n",
    "    img_embedding, text_embedding, _ = model(img_preprocessed, text_processed)\n",
    "probs = (100 * img_embedding @ text_embedding.T).softmax(dim=-1)\n",
    "plt.bar(descriptions.keys(), probs.squeeze().detach().cpu().numpy())\n",
    "plt.xticks(rotation=45)\n",
    "plt.ylabel('Probability (%)')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Before we move on to int8, lets get a feel for how fast things are in general, for a batch of size 128."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "def get_time(k, fn, repeat=32):\n",
    "\n",
    "    for _ in range(repeat // 2):\n",
    "       fn()\n",
    "\n",
    "    torch.cuda.synchronize()\n",
    "    start = time.time()\n",
    "    for _ in range(repeat):\n",
    "       fn()\n",
    "\n",
    "    torch.cuda.synchronize()\n",
    "    end = time.time()\n",
    "    ms = (end - start) / repeat * 1000\n",
    "    print(f\"time {k}: {ms:.3f} ms\")\n",
    "    return ms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time standard: 442.668 ms\n"
     ]
    }
   ],
   "source": [
    "fake_img_batch = torch.randn(128, 3, 224, 224).cuda()\n",
    "fake_txt_batch = torch.randint(0, 100, (128, 77)).cuda()\n",
    "model.set_grad_checkpointing()\n",
    "model.eval()\n",
    "def fake_forward(model, fake_img_batch, fake_txt_batch):\n",
    "    with torch.cuda.amp.autocast():\n",
    "        model(fake_img_batch, fake_txt_batch)\n",
    "time_standard = get_time(\"standard\", lambda: fake_forward(model, fake_img_batch, fake_txt_batch))\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's repeat the previous steps but change the linear layer operations to quantized in8."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================================BUG REPORT===================================\n",
      "Welcome to bitsandbytes. For bug reports, please run\n",
      "\n",
      "python -m bitsandbytes\n",
      "\n",
      " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
      "================================================================================\n",
      "bin /admin/home-mitchellw/git/bitsandbytes/bitsandbytes/libbitsandbytes_cuda118.so\n",
      "CUDA SETUP: CUDA runtime path found: /fsx/home-mitchellw/miniconda3/envs/py2/lib/libcudart.so.11.0\n",
      "CUDA SETUP: Highest compute capability among GPUs detected: 8.0\n",
      "CUDA SETUP: Detected CUDA version 118\n",
      "CUDA SETUP: Loading binary /admin/home-mitchellw/git/bitsandbytes/bitsandbytes/libbitsandbytes_cuda118.so...\n"
     ]
    }
   ],
   "source": [
    "# This replaces linear layers with int8_linear_layer\n",
    "import bitsandbytes as bnb\n",
    "model = model.cpu()\n",
    "int8_linear_layer = bnb.nn.triton_based_modules.SwitchBackLinear\n",
    "# replace linear layers, for now just replace FFN - more coming in later PR\n",
    "int8_model = open_clip.utils.replace_linear(model, int8_linear_layer, include_modules=['c_fc', 'c_proj']).cuda()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time int8-v1: 391.433 ms\n"
     ]
    }
   ],
   "source": [
    "# repeat the speed test. may take a sec to run this because it has to compile the kernels.\n",
    "int8_model.set_grad_checkpointing()\n",
    "int8_model.eval()\n",
    "time_int8_v1 = get_time(\"int8-v1\", lambda: fake_forward(int8_model, fake_img_batch, fake_txt_batch))\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you just care about inference you can make things go faster by precomputing the int8 quantized weights and deleting the original weights. This is what you should do if you're not training as it's also much less memory. We'll explain later in more detail.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n",
      "=> preparing for eval.\n"
     ]
    }
   ],
   "source": [
    "# prepare for eval by deleting the original weights and storing the quantized version of the weights\n",
    "from open_clip.utils import convert_int8_model_to_inference_mode\n",
    "convert_int8_model_to_inference_mode(int8_model)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time int8-v2: 382.608 ms\n"
     ]
    }
   ],
   "source": [
    "# Repeat speed test but with pre-quantized weights\n",
    "time_int8_v2 = get_time(\"int8-v2\", lambda: fake_forward(int8_model, fake_img_batch, fake_txt_batch))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's make sure the model still works"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 29236 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# lets make sure it still works\n",
    "with torch.cuda.amp.autocast():\n",
    "    img_embedding, text_embedding, _ = int8_model(img_preprocessed, text_processed)\n",
    "probs = (100 * img_embedding @ text_embedding.T).softmax(dim=-1)\n",
    "plt.bar(descriptions.keys(), probs.squeeze().detach().cpu().numpy())\n",
    "plt.xticks(rotation=45)\n",
    "plt.ylabel('Probability (%)')\n",
    "plt.show()\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Nice! However, our int8 linear layer actually uses the PyTorch autograd python module, whereas the default PyTorch linear layer contains C++ optimizations...\n",
    "\n",
    "So to understand the true speed-ups lets look at how fast the fair baseline is, which is a standard linear layer implemented with autograd."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "del model, int8_model\n",
    "model, _, preprocess = open_clip.create_model_and_transforms('ViT-H-14', pretrained='laion2b_s32b_b79k')\n",
    "model = open_clip.utils.replace_linear(model, bnb.nn.triton_based_modules.StandardLinear, include_modules=['c_fc', 'c_proj']).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time baseline: 521.073 ms\n"
     ]
    }
   ],
   "source": [
    "# how fast is this baseline which uses the autograd linear?\n",
    "model.set_grad_checkpointing()\n",
    "model.eval()\n",
    "time_baseline = get_time(\"baseline\", lambda: fake_forward(model, fake_img_batch, fake_txt_batch))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's compare end-to-end speed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 35320 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "speedup over autograd linear 24.88%\n",
      "speedup over optimized linear 11.57%\n"
     ]
    }
   ],
   "source": [
    "# now lets compare the speed-ups!\n",
    "labels = ['With baseline autograd linear', 'With optimized PyTorch linear', 'With our autograd int8 linear', 'With our autograd int8 linear\\nand pre-qauntized weights']\n",
    "data = [time_baseline, time_standard, time_int8_v1, time_int8_v2]\n",
    "labels.reverse()\n",
    "data.reverse()\n",
    "plt.barh(labels, data)\n",
    "plt.title('End-to-end speed comparison')\n",
    "plt.xlabel('milliseconds')\n",
    "plt.show()\n",
    "print('speedup over autograd linear {:.2f}%'.format(100 * (time_baseline - time_int8_v1) / time_baseline))\n",
    "print('speedup over optimized linear {:.2f}%'.format(100 * (time_standard - time_int8_v1) / time_standard))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How does it work? Let's go through the steps of a normal and quantized forward pass with batch X [b, n] and weights W [m, n]\n",
    "\n",
    "Starting with a normal forward pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "X = torch.randn(128, 1024).half().cuda()\n",
    "W = (1/np.sqrt(1024)) * torch.randn(4096, 1024).half().cuda()\n",
    "\n",
    "# here is a standard forward pass\n",
    "standard_out = torch.matmul(X, W.t())"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's look at a quantized forward pass, for now lets assume that the quantized weights are not pre-computed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Difference due to quantization 0.01125335693359375\n"
     ]
    }
   ],
   "source": [
    "# This operation quantizes and casts X to int8. Each row is quantized independently by dividing by the absolute max value in the row then multiplying by 127 and roundnig.\n",
    "# state_X stores the absolute max of each row\n",
    "X_int8, state_X = bnb.nn.triton_based_modules.quantize_rowwise(X)\n",
    "\n",
    "# For the weights we do tensor-wise quantization, so we quantize using the absolute max of the entire tensor.\n",
    "# During inference weights can be \"pre-quantized\" so this step does not happen -- this is what \"prepare_for_eval\" did above.\n",
    "W_int8, state_W = bnb.nn.triton_based_modules.quantize_global(W)\n",
    "\n",
    "# Now we do a fused matmul and dequantize operation\n",
    "int8_out = bnb.nn.triton_based_modules.int8_matmul_mixed_dequanitze(X_int8, W_int8.t(), state_X, state_W, None)\n",
    "\n",
    "# comparing the difference\n",
    "print('Difference due to quantization', (int8_out - standard_out).abs().mean().item())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py2",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
