{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.6"
    },
    "colab": {
      "name": "yamnet_visualization.ipynb",
      "provenance": []
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "xcZmKVHusxQT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Copyright 2019 The TensorFlow Authors All Rights Reserved.\n",
        "#\n",
        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License.\n",
        "# =============================================================================="
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0sDpjNbksxQa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Visualization of the YAMNet audio event classification model.\n",
        "# See https://github.com/tensorflow/models/tree/master/research/audioset/yamnet/\n",
        "#\n",
        "# This notebook can be run in Google Colab at https://colab.research.google.com\n",
        "# by either downloading this ipynb and uploading it, or by looking up the\n",
        "# notebook directly on GitHub in Colab's \"Open notebook\" dialog."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hnI3jFRHs-N7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 521
        },
        "outputId": "aeaef1aa-0c6d-4e43-de30-cfe6dde28910"
      },
      "source": [
        "# Install required packages.\n",
        "!pip install soundfile\n",
        "!git clone https://github.com/tensorflow/models.git\n",
        "%cd models/research/audioset/yamnet\n",
        "\n",
        "# Download YAMNet data\n",
        "!curl -O https://storage.googleapis.com/audioset/yamnet.h5\n",
        "\n",
        "# Download audio for testing\n",
        "!curl -O https://storage.googleapis.com/audioset/speech_whistling2.wav\n",
        "\n",
        "!ls -l"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: soundfile in /usr/local/lib/python3.6/dist-packages (0.10.3.post1)\n",
            "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.6/dist-packages (from soundfile) (1.14.1)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.6/dist-packages (from cffi>=1.0->soundfile) (2.20)\n",
            "Cloning into 'models'...\n",
            "remote: Enumerating objects: 34, done.\u001b[K\n",
            "remote: Counting objects: 100% (34/34), done.\u001b[K\n",
            "remote: Compressing objects: 100% (33/33), done.\u001b[K\n",
            "remote: Total 41244 (delta 12), reused 21 (delta 0), pack-reused 41210\u001b[K\n",
            "Receiving objects: 100% (41244/41244), 548.96 MiB | 34.16 MiB/s, done.\n",
            "Resolving deltas: 100% (27926/27926), done.\n",
            "/content/models/research/audioset/yamnet/models/research/audioset/yamnet/models/research/audioset/yamnet\n",
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100 14.5M  100 14.5M    0     0  27.1M      0 --:--:-- --:--:-- --:--:-- 27.1M\n",
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100  153k  100  153k    0     0   449k      0 --:--:-- --:--:-- --:--:--  449k\n",
            "total 15280\n",
            "-rw-r--r-- 1 root root     3358 Aug 11 22:54 features.py\n",
            "-rw-r--r-- 1 root root     2406 Aug 11 22:54 inference.py\n",
            "-rw-r--r-- 1 root root     1562 Aug 11 22:54 params.py\n",
            "-rw-r--r-- 1 root root     5012 Aug 11 22:54 README.md\n",
            "-rw-r--r-- 1 root root   157484 Aug 11 22:54 speech_whistling2.wav\n",
            "-rw-r--r-- 1 root root    14096 Aug 11 22:54 yamnet_class_map.csv\n",
            "-rw-r--r-- 1 root root 15296092 Aug 11 22:54 yamnet.h5\n",
            "-rw-r--r-- 1 root root     5431 Aug 11 22:54 yamnet.py\n",
            "-rw-r--r-- 1 root root     2372 Aug 11 22:54 yamnet_test.py\n",
            "-rw-r--r-- 1 root root   136563 Aug 11 22:54 yamnet_visualization.ipynb\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M0woGtbhsxQg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Imports.\n",
        "import numpy as np\n",
        "import soundfile as sf\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import params\n",
        "import yamnet as yamnet_model\n",
        "import tensorflow as tf"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jt2v3i94sxQl",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "7c2a7b4f-cca6-4049-f54c-174a94ce226f"
      },
      "source": [
        "# Read in the audio.\n",
        "wav_file_name = 'speech_whistling2.wav'\n",
        "wav_data, sr = sf.read(wav_file_name, dtype=np.int16)\n",
        "waveform = wav_data / 32768.0\n",
        "# The graph is designed for a sampling rate of 16 kHz, but higher rates \n",
        "# should work too.\n",
        "params.SAMPLE_RATE = sr\n",
        "print(\"Sample rate =\", params.SAMPLE_RATE)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sample rate = 16000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bHNJU9JUsxQs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Set up the YAMNet model.\n",
        "class_names = yamnet_model.class_names('yamnet_class_map.csv')\n",
        "params.PATCH_HOP_SECONDS = 0.1  # 10 Hz scores frame rate.\n",
        "yamnet = yamnet_model.yamnet_frames_model(params)\n",
        "yamnet.load_weights('yamnet.h5')"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XCrhG2WrsxQx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Run the model.\n",
        "scores, embeddings, spectrogram = yamnet(waveform)\n",
        "scores = scores.numpy()\n",
        "spectrogram = spectrogram.numpy()"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XN67xLQasxQ2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 483
        },
        "outputId": "b38f68eb-082d-4cdf-cde8-829a47ad3c9c"
      },
      "source": [
        "# Visualize the results.\n",
        "plt.figure(figsize=(10, 8))\n",
        "\n",
        "# Plot the waveform.\n",
        "plt.subplot(3, 1, 1)\n",
        "plt.plot(waveform)\n",
        "plt.xlim([0, len(waveform)])\n",
        "# Plot the log-mel spectrogram (returned by the model).\n",
        "plt.subplot(3, 1, 2)\n",
        "plt.imshow(spectrogram.T, aspect='auto', interpolation='nearest', origin='bottom')\n",
        "\n",
        "# Plot and label the model output scores for the top-scoring classes.\n",
        "mean_scores = np.mean(scores, axis=0)\n",
        "top_N = 10\n",
        "top_class_indices = np.argsort(mean_scores)[::-1][:top_N]\n",
        "plt.subplot(3, 1, 3)\n",
        "plt.imshow(scores[:, top_class_indices].T, aspect='auto', interpolation='nearest', cmap='gray_r')\n",
        "# Compensate for the PATCH_WINDOW_SECONDS (0.96 s) context window to align with spectrogram.\n",
        "patch_padding = (params.PATCH_WINDOW_SECONDS / 2) / params.PATCH_HOP_SECONDS\n",
        "plt.xlim([-patch_padding, scores.shape[0] + patch_padding])\n",
        "# Label the top_N classes.\n",
        "yticks = range(0, top_N, 1)\n",
        "plt.yticks(yticks, [class_names[top_class_indices[x]] for x in yticks])\n",
        "_ = plt.ylim(-0.5 + np.array([top_N, 0]))\n"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "[encoded data, 130644 characters]\n",
            "text/plain": [
              "<Figure size 720x576 with 3 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vQmjVIylviFq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}
