Resolve conflicts for #4760 (#4762)

docs/source/tutorials/hpo_quickstart_pytorch/model.py.md5

+ed8bfc27e3d555d842fc4eec2635e619
\ No newline at end of file

docs/source/tutorials/hpo_quickstart_pytorch/model.rst

+:orphan:
+
+.. DO NOT EDIT.
+.. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY.
+.. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE:
+.. "tutorials/hpo_quickstart_pytorch/model.py"
+.. LINE NUMBERS ARE GIVEN BELOW.
+
+.. only:: html
+
+    .. note::
+        :class: sphx-glr-download-link-note
+
+        Click :ref:`here <sphx_glr_download_tutorials_hpo_quickstart_pytorch_model.py>`
+        to download the full example code
+
+.. rst-class:: sphx-glr-example-title
+
+.. _sphx_glr_tutorials_hpo_quickstart_pytorch_model.py:
+
+
+Port PyTorch Quickstart to NNI
+==============================
+This is a modified version of `PyTorch quickstart`_.
+
+It can be run directly and will have the exact same result as original version.
+
+Furthermore, it enables the ability of auto tuning with an NNI *experiment*, which will be detailed later.
+
+It is recommended to run this script directly first to verify the environment.
+
+There are 2 key differences from the original version:
+
+1. In `Get optimized hyperparameters`_ part, it receives generated hyperparameters.
+2. In `Train model and report accuracy`_ part, it reports accuracy metrics to NNI.
+
+.. _PyTorch quickstart: https://pytorch.org/tutorials/beginner/basics/quickstart_tutorial.html
+
+.. GENERATED FROM PYTHON SOURCE LINES 21-28
+
+.. code-block:: default
+
+    import nni
+    import torch
+    from torch import nn
+    from torch.utils.data import DataLoader
+    from torchvision import datasets
+    from torchvision.transforms import ToTensor
+
+
+
+
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 29-32
+
+Hyperparameters to be tuned
+---------------------------
+These are the hyperparameters that will be tuned.
+
+.. GENERATED FROM PYTHON SOURCE LINES 32-38
+
+.. code-block:: default
+
+    params = {
+        'features': 512,
+        'lr': 0.001,
+        'momentum': 0,
+    }
+
+
+
+
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 39-43
+
+Get optimized hyperparameters
+-----------------------------
+If run directly, :func:`nni.get_next_parameter` is a no-op and returns an empty dict.
+But with an NNI *experiment*, it will receive optimized hyperparameters from tuning algorithm.
+
+.. GENERATED FROM PYTHON SOURCE LINES 43-47
+
+.. code-block:: default
+
+    optimized_params = nni.get_next_parameter()
+    params.update(optimized_params)
+    print(params)
+
+
+
+
+
+.. rst-class:: sphx-glr-script-out
+
+ Out:
+
+ .. code-block:: none
+
+    {'features': 512, 'lr': 0.001, 'momentum': 0}
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 48-50
+
+Load dataset
+------------
+
+.. GENERATED FROM PYTHON SOURCE LINES 50-58
+
+.. code-block:: default
+
+    training_data = datasets.FashionMNIST(root="data", train=True, download=True, transform=ToTensor())
+    test_data = datasets.FashionMNIST(root="data", train=False, download=True, transform=ToTensor())
+
+    batch_size = 64
+
+    train_dataloader = DataLoader(training_data, batch_size=batch_size)
+    test_dataloader = DataLoader(test_data, batch_size=batch_size)
+
+
+
+
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 59-61
+
+Build model with hyperparameters
+--------------------------------
+
+.. GENERATED FROM PYTHON SOURCE LINES 61-86
+
+.. code-block:: default
+
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    print(f"Using {device} device")
+
+    class NeuralNetwork(nn.Module):
+        def __init__(self):
+            super(NeuralNetwork, self).__init__()
+            self.flatten = nn.Flatten()
+            self.linear_relu_stack = nn.Sequential(
+                nn.Linear(28*28, params['features']),
+                nn.ReLU(),
+                nn.Linear(params['features'], params['features']),
+                nn.ReLU(),
+                nn.Linear(params['features'], 10)
+            )
+
+        def forward(self, x):
+            x = self.flatten(x)
+            logits = self.linear_relu_stack(x)
+            return logits
+
+    model = NeuralNetwork().to(device)
+
+    loss_fn = nn.CrossEntropyLoss()
+    optimizer = torch.optim.SGD(model.parameters(), lr=params['lr'], momentum=params['momentum'])
+
+
+
+
+
+.. rst-class:: sphx-glr-script-out
+
+ Out:
+
+ .. code-block:: none
+
+    Using cpu device
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 87-89
+
+Define train and test
+---------------------
+
+.. GENERATED FROM PYTHON SOURCE LINES 89-115
+
+.. code-block:: default
+
+    def train(dataloader, model, loss_fn, optimizer):
+        size = len(dataloader.dataset)
+        model.train()
+        for batch, (X, y) in enumerate(dataloader):
+            X, y = X.to(device), y.to(device)
+            pred = model(X)
+            loss = loss_fn(pred, y)
+            optimizer.zero_grad()
+            loss.backward()
+            optimizer.step()
+
+    def test(dataloader, model, loss_fn):
+        size = len(dataloader.dataset)
+        num_batches = len(dataloader)
+        model.eval()
+        test_loss, correct = 0, 0
+        with torch.no_grad():
+            for X, y in dataloader:
+                X, y = X.to(device), y.to(device)
+                pred = model(X)
+                test_loss += loss_fn(pred, y).item()
+                correct += (pred.argmax(1) == y).type(torch.float).sum().item()
+        test_loss /= num_batches
+        correct /= size
+        return correct
+
+
+
+
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 116-119
+
+Train model and report accuracy
+-------------------------------
+Report accuracy metrics to NNI so the tuning algorithm can suggest better hyperparameters.
+
+.. GENERATED FROM PYTHON SOURCE LINES 119-126
+
+.. code-block:: default
+
+    epochs = 5
+    for t in range(epochs):
+        print(f"Epoch {t+1}\n-------------------------------")
+        train(train_dataloader, model, loss_fn, optimizer)
+        accuracy = test(test_dataloader, model, loss_fn)
+        nni.report_intermediate_result(accuracy)
+    nni.report_final_result(accuracy)
+
+
+
+
+.. rst-class:: sphx-glr-script-out
+
+ Out:
+
+ .. code-block:: none
+
+    Epoch 1
+    -------------------------------
+    [2022-03-21 01:09:37] INFO (nni/MainThread) Intermediate result: 0.461  (Index 0)
+    Epoch 2
+    -------------------------------
+    [2022-03-21 01:09:42] INFO (nni/MainThread) Intermediate result: 0.5529  (Index 1)
+    Epoch 3
+    -------------------------------
+    [2022-03-21 01:09:47] INFO (nni/MainThread) Intermediate result: 0.6155  (Index 2)
+    Epoch 4
+    -------------------------------
+    [2022-03-21 01:09:52] INFO (nni/MainThread) Intermediate result: 0.6345  (Index 3)
+    Epoch 5
+    -------------------------------
+    [2022-03-21 01:09:56] INFO (nni/MainThread) Intermediate result: 0.6505  (Index 4)
+    [2022-03-21 01:09:56] INFO (nni/MainThread) Final result: 0.6505
+
+
+
+
+
+.. rst-class:: sphx-glr-timing
+
+   **Total running time of the script:** ( 0 minutes  24.441 seconds)
+
+
+.. _sphx_glr_download_tutorials_hpo_quickstart_pytorch_model.py:
+
+
+.. only :: html
+
+ .. container:: sphx-glr-footer
+    :class: sphx-glr-footer-example
+
+
+
+  .. container:: sphx-glr-download sphx-glr-download-python
+
+     :download:`Download Python source code: model.py <model.py>`
+
+
+
+  .. container:: sphx-glr-download sphx-glr-download-jupyter
+
+     :download:`Download Jupyter notebook: model.ipynb <model.ipynb>`
+
+
+.. only:: html
+
+ .. rst-class:: sphx-glr-signature
+
+    `Gallery generated by Sphinx-Gallery <https://sphinx-gallery.github.io>`_

docs/source/tutorials/hpo_quickstart_pytorch/model_zh.rst

+.. e083b4dc8e350428ddf680e97b47cc8e
+
+:orphan:
+
+.. DO NOT EDIT.
+.. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY.
+.. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE:
+.. "tutorials/hpo_quickstart_pytorch/model.py"
+.. LINE NUMBERS ARE GIVEN BELOW.
+
+.. only:: html
+
+    .. note::
+        :class: sphx-glr-download-link-note
+
+        Click :ref:`here <sphx_glr_download_tutorials_hpo_quickstart_pytorch_model.py>`
+        to download the full example code
+
+.. rst-class:: sphx-glr-example-title
+
+.. _sphx_glr_tutorials_hpo_quickstart_pytorch_model.py:
+
+将 PyTorch 官方教程移植到NNI
+============================
+本文件是 `PyTorch 官方教程 <https://pytorch.org/tutorials/beginner/basics/quickstart_tutorial.htlm>`__ 的修改版。
+
+您可以直接运行本文件，其结果和原版完全一致。同时，您也可以在 NNI 实验中使用本文件，进行超参调优。
+
+我们建议您先直接运行一次本文件，在熟悉代码的同时检查运行环境。
+
+和原版相比，我们做了两处修改：
+
+1. 在 `获取调优后的参数`_ 部分，我们使用调优算法生成的参数替换默认参数；
+2. 在 `训练模型并上传结果`_ 部分，我们将准确率数据报告给 NNI。
+
+.. GENERATED FROM PYTHON SOURCE LINES 21-28
+
+.. code-block:: default
+
+    import nni
+    import torch
+    from torch import nn
+    from torch.utils.data import DataLoader
+    from torchvision import datasets
+    from torchvision.transforms import ToTensor
+
+
+
+
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 29-32
+
+准备调优的超参
+--------------
+以下超参将被调优：
+
+.. GENERATED FROM PYTHON SOURCE LINES 32-38
+
+.. code-block:: default
+
+    params = {
+        'features': 512,
+        'lr': 0.001,
+        'momentum': 0,
+    }
+
+
+
+
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 39-43
+
+获取调优后的参数
+----------------
+直接运行时 :func:`nni.get_next_parameter` 会返回空 dict，
+而在 NNI 实验中使用时，它会返回调优算法生成的超参组合。
+
+.. GENERATED FROM PYTHON SOURCE LINES 43-47
+
+.. code-block:: default
+
+    optimized_params = nni.get_next_parameter()
+    params.update(optimized_params)
+    print(params)
+
+
+
+
+
+.. rst-class:: sphx-glr-script-out
+
+ Out:
+
+ .. code-block:: none
+
+    {'features': 512, 'lr': 0.001, 'momentum': 0}
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 48-50
+
+加载数据集
+----------
+
+.. GENERATED FROM PYTHON SOURCE LINES 50-58
+
+.. code-block:: default
+
+    training_data = datasets.FashionMNIST(root="data", train=True, download=True, transform=ToTensor())
+    test_data = datasets.FashionMNIST(root="data", train=False, download=True, transform=ToTensor())
+
+    batch_size = 64
+
+    train_dataloader = DataLoader(training_data, batch_size=batch_size)
+    test_dataloader = DataLoader(test_data, batch_size=batch_size)
+
+
+
+
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 59-61
+
+使用超参构建模型
+----------------
+
+.. GENERATED FROM PYTHON SOURCE LINES 61-86
+
+.. code-block:: default
+
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    print(f"Using {device} device")
+
+    class NeuralNetwork(nn.Module):
+        def __init__(self):
+            super(NeuralNetwork, self).__init__()
+            self.flatten = nn.Flatten()
+            self.linear_relu_stack = nn.Sequential(
+                nn.Linear(28*28, params['features']),
+                nn.ReLU(),
+                nn.Linear(params['features'], params['features']),
+                nn.ReLU(),
+                nn.Linear(params['features'], 10)
+            )
+
+        def forward(self, x):
+            x = self.flatten(x)
+            logits = self.linear_relu_stack(x)
+            return logits
+
+    model = NeuralNetwork().to(device)
+
+    loss_fn = nn.CrossEntropyLoss()
+    optimizer = torch.optim.SGD(model.parameters(), lr=params['lr'], momentum=params['momentum'])
+
+
+
+
+
+.. rst-class:: sphx-glr-script-out
+
+ Out:
+
+ .. code-block:: none
+
+    Using cpu device
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 87-89
+
+定义训练和测试函数
+------------------
+
+.. GENERATED FROM PYTHON SOURCE LINES 89-115
+
+.. code-block:: default
+
+    def train(dataloader, model, loss_fn, optimizer):
+        size = len(dataloader.dataset)
+        model.train()
+        for batch, (X, y) in enumerate(dataloader):
+            X, y = X.to(device), y.to(device)
+            pred = model(X)
+            loss = loss_fn(pred, y)
+            optimizer.zero_grad()
+            loss.backward()
+            optimizer.step()
+
+    def test(dataloader, model, loss_fn):
+        size = len(dataloader.dataset)
+        num_batches = len(dataloader)
+        model.eval()
+        test_loss, correct = 0, 0
+        with torch.no_grad():
+            for X, y in dataloader:
+                X, y = X.to(device), y.to(device)
+                pred = model(X)
+                test_loss += loss_fn(pred, y).item()
+                correct += (pred.argmax(1) == y).type(torch.float).sum().item()
+        test_loss /= num_batches
+        correct /= size
+        return correct
+
+
+
+
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 116-119
+
+训练模型并上传结果
+------------------
+将准确率数据报告给 NNI 的调参算法，以使其能够预测更优的超参组合。
+
+.. GENERATED FROM PYTHON SOURCE LINES 119-126
+
+.. code-block:: default
+
+    epochs = 5
+    for t in range(epochs):
+        print(f"Epoch {t+1}\n-------------------------------")
+        train(train_dataloader, model, loss_fn, optimizer)
+        accuracy = test(test_dataloader, model, loss_fn)
+        nni.report_intermediate_result(accuracy)
+    nni.report_final_result(accuracy)
+
+
+
+
+.. rst-class:: sphx-glr-script-out
+
+ Out:
+
+ .. code-block:: none
+
+    Epoch 1
+    -------------------------------
+    [2022-03-21 01:09:37] INFO (nni/MainThread) Intermediate result: 0.461  (Index 0)
+    Epoch 2
+    -------------------------------
+    [2022-03-21 01:09:42] INFO (nni/MainThread) Intermediate result: 0.5529  (Index 1)
+    Epoch 3
+    -------------------------------
+    [2022-03-21 01:09:47] INFO (nni/MainThread) Intermediate result: 0.6155  (Index 2)
+    Epoch 4
+    -------------------------------
+    [2022-03-21 01:09:52] INFO (nni/MainThread) Intermediate result: 0.6345  (Index 3)
+    Epoch 5
+    -------------------------------
+    [2022-03-21 01:09:56] INFO (nni/MainThread) Intermediate result: 0.6505  (Index 4)
+    [2022-03-21 01:09:56] INFO (nni/MainThread) Final result: 0.6505
+
+
+
+
+
+.. rst-class:: sphx-glr-timing
+
+   **Total running time of the script:** ( 0 minutes  24.441 seconds)
+
+
+.. _sphx_glr_download_tutorials_hpo_quickstart_pytorch_model.py:
+
+
+.. only :: html
+
+ .. container:: sphx-glr-footer
+    :class: sphx-glr-footer-example
+
+
+
+  .. container:: sphx-glr-download sphx-glr-download-python
+
+     :download:`Download Python source code: model.py <model.py>`
+
+
+
+  .. container:: sphx-glr-download sphx-glr-download-jupyter
+
+     :download:`Download Jupyter notebook: model.ipynb <model.ipynb>`
+
+
+.. only:: html
+
+ .. rst-class:: sphx-glr-signature
+
+    `Gallery generated by Sphinx-Gallery <https://sphinx-gallery.github.io>`_

docs/source/tutorials/hpo_quickstart_pytorch/sg_execution_times.rst

+
+:orphan:
+
+.. _sphx_glr_tutorials_hpo_quickstart_pytorch_sg_execution_times:
+
+Computation times
+=================
+**01:24.367** total execution time for **tutorials_hpo_quickstart_pytorch** files:
+
++--------------------------------------------------------------------------+-----------+--------+
+| :ref:`sphx_glr_tutorials_hpo_quickstart_pytorch_main.py` (``main.py``)   | 01:24.367 | 0.0 MB |
++--------------------------------------------------------------------------+-----------+--------+
+| :ref:`sphx_glr_tutorials_hpo_quickstart_pytorch_model.py` (``model.py``) | 00:00.000 | 0.0 MB |
++--------------------------------------------------------------------------+-----------+--------+

docs/source/tutorials/hpo_quickstart_tensorflow/main.ipynb

+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "%matplotlib inline"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "\n# HPO Quickstart with TensorFlow\nThis tutorial optimizes the model in `official TensorFlow quickstart`_ with auto-tuning.\n\nThe tutorial consists of 4 steps: \n\n1. Modify the model for auto-tuning.\n2. Define hyperparameters' search space.\n3. Configure the experiment.\n4. Run the experiment.\n\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Step 1: Prepare the model\nIn first step, we need to prepare the model to be tuned.\n\nThe model should be put in a separate script.\nIt will be evaluated many times concurrently,\nand possibly will be trained on distributed platforms.\n\nIn this tutorial, the model is defined in :doc:`model.py <model>`.\n\nIn short, it is a TensorFlow model with 3 additional API calls:\n\n1. Use :func:`nni.get_next_parameter` to fetch the hyperparameters to be evalutated.\n2. Use :func:`nni.report_intermediate_result` to report per-epoch accuracy metrics.\n3. Use :func:`nni.report_final_result` to report final accuracy.\n\nPlease understand the model code before continue to next step.\n\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Step 2: Define search space\nIn model code, we have prepared 4 hyperparameters to be tuned:\n*dense_units*, *activation_type*, *dropout_rate*, and *learning_rate*.\n\nHere we need to define their *search space* so the tuning algorithm can sample them in desired range.\n\nAssuming we have following prior knowledge for these hyperparameters:\n\n1. *dense_units* should be one of 64, 128, 256.\n2. *activation_type* should be one of 'relu', 'tanh', 'swish', or None.\n3. *dropout_rate* should be a float between 0.5 and 0.9.\n4. *learning_rate* should be a float between 0.0001 and 0.1, and it follows exponential distribution.\n\nIn NNI, the space of *dense_units* and *activation_type* is called ``choice``;\nthe space of *dropout_rate* is called ``uniform``;\nand the space of *learning_rate* is called ``loguniform``.\nYou may have noticed, these names are derived from ``numpy.random``.\n\nFor full specification of search space, check :doc:`the reference </hpo/search_space>`.\n\nNow we can define the search space as follow:\n\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "search_space = {\n    'dense_units': {'_type': 'choice', '_value': [64, 128, 256]},\n    'activation_type': {'_type': 'choice', '_value': ['relu', 'tanh', 'swish', None]},\n    'dropout_rate': {'_type': 'uniform', '_value': [0.5, 0.9]},\n    'learning_rate': {'_type': 'loguniform', '_value': [0.0001, 0.1]},\n}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Step 3: Configure the experiment\nNNI uses an *experiment* to manage the HPO process.\nThe *experiment config* defines how to train the models and how to explore the search space.\n\nIn this tutorial we use a *local* mode experiment,\nwhich means models will be trained on local machine, without using any special training platform.\n\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "from nni.experiment import Experiment\nexperiment = Experiment('local')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "Now we start to configure the experiment.\n\n### Configure trial code\nIn NNI evaluation of each hyperparameter set is called a *trial*.\nSo the model script is called *trial code*.\n\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "experiment.config.trial_command = 'python model.py'\nexperiment.config.trial_code_directory = '.'"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "When ``trial_code_directory`` is a relative path, it relates to current working directory.\nTo run ``main.py`` in a different path, you can set trial code directory to ``Path(__file__).parent``.\n(`__file__ <https://docs.python.org/3.10/reference/datamodel.html#index-43>`__\nis only available in standard Python, not in Jupyter Notebook.)\n\n.. attention::\n\n    If you are using Linux system without Conda,\n    you may need to change ``\"python model.py\"`` to ``\"python3 model.py\"``.\n\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Configure search space\n\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "experiment.config.search_space = search_space"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Configure tuning algorithm\nHere we use :doc:`TPE tuner </hpo/tuners>`.\n\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "experiment.config.tuner.name = 'TPE'\nexperiment.config.tuner.class_args['optimize_mode'] = 'maximize'"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Configure how many trials to run\nHere we evaluate 10 sets of hyperparameters in total, and concurrently evaluate 2 sets at a time.\n\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "experiment.config.max_trial_number = 10\nexperiment.config.trial_concurrency = 2"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "You may also set ``max_experiment_duration = '1h'`` to limit running time.\n\nIf neither ``max_trial_number`` nor ``max_experiment_duration`` are set,\nthe experiment will run forever until you press Ctrl-C.\n\n<div class=\"alert alert-info\"><h4>Note</h4><p>``max_trial_number`` is set to 10 here for a fast example.\n    In real world it should be set to a larger number.\n    With default config TPE tuner requires 20 trials to warm up.</p></div>\n\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## Step 4: Run the experiment\nNow the experiment is ready. Choose a port and launch it. (Here we use port 8080.)\n\nYou can use the web portal to view experiment status: http://localhost:8080.\n\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "experiment.run(8080)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "## After the experiment is done\nEverything is done and it is safe to exit now. The following are optional.\n\nIf you are using standard Python instead of Jupyter Notebook,\nyou can add ``input()`` or ``signal.pause()`` to prevent Python from exiting,\nallowing you to view the web portal after the experiment is done.\n\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "# input('Press enter to quit')\nexperiment.stop()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        ":meth:`nni.experiment.Experiment.stop` is automatically invoked when Python exits,\nso it can be omitted in your code.\n\nAfter the experiment is stopped, you can run :meth:`nni.experiment.Experiment.view` to restart web portal.\n\n.. tip::\n\n    This example uses :doc:`Python API </reference/experiment>` to create experiment.\n\n    You can also create and manage experiments with :doc:`command line tool <../hpo_nnictl/nnictl>`.\n\n"
+      ]
+    }
+  ],
+  "metadata": {
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.10.4"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
\ No newline at end of file

docs/source/tutorials/hpo_quickstart_tensorflow/main.py

+"""
+HPO Quickstart with TensorFlow
+==============================
+This tutorial optimizes the model in `official TensorFlow quickstart`_ with auto-tuning.
+
+The tutorial consists of 4 steps: 
+
+1. Modify the model for auto-tuning.
+2. Define hyperparameters' search space.
+3. Configure the experiment.
+4. Run the experiment.
+
+.. _official TensorFlow quickstart: https://www.tensorflow.org/tutorials/quickstart/beginner
+"""
+
+# %%
+# Step 1: Prepare the model
+# -------------------------
+# In first step, we need to prepare the model to be tuned.
+#
+# The model should be put in a separate script.
+# It will be evaluated many times concurrently,
+# and possibly will be trained on distributed platforms.
+#
+# In this tutorial, the model is defined in :doc:`model.py <model>`.
+#
+# In short, it is a TensorFlow model with 3 additional API calls:
+#
+# 1. Use :func:`nni.get_next_parameter` to fetch the hyperparameters to be evalutated.
+# 2. Use :func:`nni.report_intermediate_result` to report per-epoch accuracy metrics.
+# 3. Use :func:`nni.report_final_result` to report final accuracy.
+#
+# Please understand the model code before continue to next step.
+
+# %%
+# Step 2: Define search space
+# ---------------------------
+# In model code, we have prepared 4 hyperparameters to be tuned:
+# *dense_units*, *activation_type*, *dropout_rate*, and *learning_rate*.
+#
+# Here we need to define their *search space* so the tuning algorithm can sample them in desired range.
+#
+# Assuming we have following prior knowledge for these hyperparameters:
+#
+# 1. *dense_units* should be one of 64, 128, 256.
+# 2. *activation_type* should be one of 'relu', 'tanh', 'swish', or None.
+# 3. *dropout_rate* should be a float between 0.5 and 0.9.
+# 4. *learning_rate* should be a float between 0.0001 and 0.1, and it follows exponential distribution.
+#
+# In NNI, the space of *dense_units* and *activation_type* is called ``choice``;
+# the space of *dropout_rate* is called ``uniform``;
+# and the space of *learning_rate* is called ``loguniform``.
+# You may have noticed, these names are derived from ``numpy.random``.
+#
+# For full specification of search space, check :doc:`the reference </hpo/search_space>`.
+#
+# Now we can define the search space as follow:
+
+search_space = {
+    'dense_units': {'_type': 'choice', '_value': [64, 128, 256]},
+    'activation_type': {'_type': 'choice', '_value': ['relu', 'tanh', 'swish', None]},
+    'dropout_rate': {'_type': 'uniform', '_value': [0.5, 0.9]},
+    'learning_rate': {'_type': 'loguniform', '_value': [0.0001, 0.1]},
+}
+
+# %%
+# Step 3: Configure the experiment
+# --------------------------------
+# NNI uses an *experiment* to manage the HPO process.
+# The *experiment config* defines how to train the models and how to explore the search space.
+# 
+# In this tutorial we use a *local* mode experiment,
+# which means models will be trained on local machine, without using any special training platform.
+from nni.experiment import Experiment
+experiment = Experiment('local')
+
+# %%
+# Now we start to configure the experiment.
+#
+# Configure trial code
+# ^^^^^^^^^^^^^^^^^^^^
+# In NNI evaluation of each hyperparameter set is called a *trial*.
+# So the model script is called *trial code*.
+experiment.config.trial_command = 'python model.py'
+experiment.config.trial_code_directory = '.'
+# %%
+# When ``trial_code_directory`` is a relative path, it relates to current working directory.
+# To run ``main.py`` in a different path, you can set trial code directory to ``Path(__file__).parent``.
+# (`__file__ <https://docs.python.org/3.10/reference/datamodel.html#index-43>`__
+# is only available in standard Python, not in Jupyter Notebook.)
+#
+# .. attention::
+#
+#     If you are using Linux system without Conda,
+#     you may need to change ``"python model.py"`` to ``"python3 model.py"``.
+
+# %%
+# Configure search space
+# ^^^^^^^^^^^^^^^^^^^^^^
+experiment.config.search_space = search_space
+
+# %%
+# Configure tuning algorithm
+# ^^^^^^^^^^^^^^^^^^^^^^^^^^
+# Here we use :doc:`TPE tuner </hpo/tuners>`.
+experiment.config.tuner.name = 'TPE'
+experiment.config.tuner.class_args['optimize_mode'] = 'maximize'
+
+# %%
+# Configure how many trials to run
+# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+# Here we evaluate 10 sets of hyperparameters in total, and concurrently evaluate 2 sets at a time.
+experiment.config.max_trial_number = 10
+experiment.config.trial_concurrency = 2
+# %%
+# You may also set ``max_experiment_duration = '1h'`` to limit running time.
+#
+# If neither ``max_trial_number`` nor ``max_experiment_duration`` are set,
+# the experiment will run forever until you press Ctrl-C.
+#
+# .. note::
+#
+#     ``max_trial_number`` is set to 10 here for a fast example.
+#     In real world it should be set to a larger number.
+#     With default config TPE tuner requires 20 trials to warm up.
+
+# %%
+# Step 4: Run the experiment
+# --------------------------
+# Now the experiment is ready. Choose a port and launch it. (Here we use port 8080.)
+#
+# You can use the web portal to view experiment status: http://localhost:8080.
+experiment.run(8080)
+
+# %%
+# After the experiment is done
+# ----------------------------
+# Everything is done and it is safe to exit now. The following are optional.
+#
+# If you are using standard Python instead of Jupyter Notebook,
+# you can add ``input()`` or ``signal.pause()`` to prevent Python from exiting,
+# allowing you to view the web portal after the experiment is done.
+
+# input('Press enter to quit')
+experiment.stop()
+
+# %%
+# :meth:`nni.experiment.Experiment.stop` is automatically invoked when Python exits,
+# so it can be omitted in your code.
+#
+# After the experiment is stopped, you can run :meth:`nni.experiment.Experiment.view` to restart web portal.
+#
+# .. tip::
+#
+#     This example uses :doc:`Python API </reference/experiment>` to create experiment.
+#
+#     You can also create and manage experiments with :doc:`command line tool <../hpo_nnictl/nnictl>`.

docs/source/tutorials/hpo_quickstart_tensorflow/main.py.md5

+b8a9880a36233005ade7a8dae6d428a8
\ No newline at end of file

docs/source/tutorials/hpo_quickstart_tensorflow/main.rst

+
+.. DO NOT EDIT.
+.. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY.
+.. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE:
+.. "tutorials/hpo_quickstart_tensorflow/main.py"
+.. LINE NUMBERS ARE GIVEN BELOW.
+
+.. only:: html
+
+    .. note::
+        :class: sphx-glr-download-link-note
+
+        Click :ref:`here <sphx_glr_download_tutorials_hpo_quickstart_tensorflow_main.py>`
+        to download the full example code
+
+.. rst-class:: sphx-glr-example-title
+
+.. _sphx_glr_tutorials_hpo_quickstart_tensorflow_main.py:
+
+
+HPO Quickstart with TensorFlow
+==============================
+This tutorial optimizes the model in `official TensorFlow quickstart`_ with auto-tuning.
+
+The tutorial consists of 4 steps: 
+
+1. Modify the model for auto-tuning.
+2. Define hyperparameters' search space.
+3. Configure the experiment.
+4. Run the experiment.
+
+.. _official TensorFlow quickstart: https://www.tensorflow.org/tutorials/quickstart/beginner
+
+.. GENERATED FROM PYTHON SOURCE LINES 17-34
+
+Step 1: Prepare the model
+-------------------------
+In first step, we need to prepare the model to be tuned.
+
+The model should be put in a separate script.
+It will be evaluated many times concurrently,
+and possibly will be trained on distributed platforms.
+
+In this tutorial, the model is defined in :doc:`model.py <model>`.
+
+In short, it is a TensorFlow model with 3 additional API calls:
+
+1. Use :func:`nni.get_next_parameter` to fetch the hyperparameters to be evalutated.
+2. Use :func:`nni.report_intermediate_result` to report per-epoch accuracy metrics.
+3. Use :func:`nni.report_final_result` to report final accuracy.
+
+Please understand the model code before continue to next step.
+
+.. GENERATED FROM PYTHON SOURCE LINES 36-58
+
+Step 2: Define search space
+---------------------------
+In model code, we have prepared 4 hyperparameters to be tuned:
+*dense_units*, *activation_type*, *dropout_rate*, and *learning_rate*.
+
+Here we need to define their *search space* so the tuning algorithm can sample them in desired range.
+
+Assuming we have following prior knowledge for these hyperparameters:
+
+1. *dense_units* should be one of 64, 128, 256.
+2. *activation_type* should be one of 'relu', 'tanh', 'swish', or None.
+3. *dropout_rate* should be a float between 0.5 and 0.9.
+4. *learning_rate* should be a float between 0.0001 and 0.1, and it follows exponential distribution.
+
+In NNI, the space of *dense_units* and *activation_type* is called ``choice``;
+the space of *dropout_rate* is called ``uniform``;
+and the space of *learning_rate* is called ``loguniform``.
+You may have noticed, these names are derived from ``numpy.random``.
+
+For full specification of search space, check :doc:`the reference </hpo/search_space>`.
+
+Now we can define the search space as follow:
+
+.. GENERATED FROM PYTHON SOURCE LINES 58-66
+
+.. code-block:: default
+
+
+    search_space = {
+        'dense_units': {'_type': 'choice', '_value': [64, 128, 256]},
+        'activation_type': {'_type': 'choice', '_value': ['relu', 'tanh', 'swish', None]},
+        'dropout_rate': {'_type': 'uniform', '_value': [0.5, 0.9]},
+        'learning_rate': {'_type': 'loguniform', '_value': [0.0001, 0.1]},
+    }
+
+
+
+
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 67-74
+
+Step 3: Configure the experiment
+--------------------------------
+NNI uses an *experiment* to manage the HPO process.
+The *experiment config* defines how to train the models and how to explore the search space.
+
+In this tutorial we use a *local* mode experiment,
+which means models will be trained on local machine, without using any special training platform.
+
+.. GENERATED FROM PYTHON SOURCE LINES 74-77
+
+.. code-block:: default
+
+    from nni.experiment import Experiment
+    experiment = Experiment('local')
+
+
+
+
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 78-84
+
+Now we start to configure the experiment.
+
+Configure trial code
+^^^^^^^^^^^^^^^^^^^^
+In NNI evaluation of each hyperparameter set is called a *trial*.
+So the model script is called *trial code*.
+
+.. GENERATED FROM PYTHON SOURCE LINES 84-86
+
+.. code-block:: default
+
+    experiment.config.trial_command = 'python model.py'
+    experiment.config.trial_code_directory = '.'
+
+
+
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 87-96
+
+When ``trial_code_directory`` is a relative path, it relates to current working directory.
+To run ``main.py`` in a different path, you can set trial code directory to ``Path(__file__).parent``.
+(`__file__ <https://docs.python.org/3.10/reference/datamodel.html#index-43>`__
+is only available in standard Python, not in Jupyter Notebook.)
+
+.. attention::
+
+    If you are using Linux system without Conda,
+    you may need to change ``"python model.py"`` to ``"python3 model.py"``.
+
+.. GENERATED FROM PYTHON SOURCE LINES 98-100
+
+Configure search space
+^^^^^^^^^^^^^^^^^^^^^^
+
+.. GENERATED FROM PYTHON SOURCE LINES 100-102
+
+.. code-block:: default
+
+    experiment.config.search_space = search_space
+
+
+
+
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 103-106
+
+Configure tuning algorithm
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+Here we use :doc:`TPE tuner </hpo/tuners>`.
+
+.. GENERATED FROM PYTHON SOURCE LINES 106-109
+
+.. code-block:: default
+
+    experiment.config.tuner.name = 'TPE'
+    experiment.config.tuner.class_args['optimize_mode'] = 'maximize'
+
+
+
+
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 110-113
+
+Configure how many trials to run
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Here we evaluate 10 sets of hyperparameters in total, and concurrently evaluate 2 sets at a time.
+
+.. GENERATED FROM PYTHON SOURCE LINES 113-115
+
+.. code-block:: default
+
+    experiment.config.max_trial_number = 10
+    experiment.config.trial_concurrency = 2
+
+
+
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 116-126
+
+You may also set ``max_experiment_duration = '1h'`` to limit running time.
+
+If neither ``max_trial_number`` nor ``max_experiment_duration`` are set,
+the experiment will run forever until you press Ctrl-C.
+
+.. note::
+
+    ``max_trial_number`` is set to 10 here for a fast example.
+    In real world it should be set to a larger number.
+    With default config TPE tuner requires 20 trials to warm up.
+
+.. GENERATED FROM PYTHON SOURCE LINES 128-133
+
+Step 4: Run the experiment
+--------------------------
+Now the experiment is ready. Choose a port and launch it. (Here we use port 8080.)
+
+You can use the web portal to view experiment status: http://localhost:8080.
+
+.. GENERATED FROM PYTHON SOURCE LINES 133-135
+
+.. code-block:: default
+
+    experiment.run(8080)
+
+
+
+
+
+.. rst-class:: sphx-glr-script-out
+
+ Out:
+
+ .. code-block:: none
+
+    [2022-04-13 12:11:34] Creating experiment, Experiment ID: enw27qxj
+    [2022-04-13 12:11:34] Starting web server...
+    [2022-04-13 12:11:35] Setting up...
+    [2022-04-13 12:11:35] Web portal URLs: http://127.0.0.1:8080 http://192.168.100.103:8080
+
+    True
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 136-143
+
+After the experiment is done
+----------------------------
+Everything is done and it is safe to exit now. The following are optional.
+
+If you are using standard Python instead of Jupyter Notebook,
+you can add ``input()`` or ``signal.pause()`` to prevent Python from exiting,
+allowing you to view the web portal after the experiment is done.
+
+.. GENERATED FROM PYTHON SOURCE LINES 143-147
+
+.. code-block:: default
+
+
+    # input('Press enter to quit')
+    experiment.stop()
+
+
+
+
+
+.. rst-class:: sphx-glr-script-out
+
+ Out:
+
+ .. code-block:: none
+
+    [2022-04-13 12:12:55] Stopping experiment, please wait...
+    [2022-04-13 12:12:58] Experiment stopped
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 148-158
+
+:meth:`nni.experiment.Experiment.stop` is automatically invoked when Python exits,
+so it can be omitted in your code.
+
+After the experiment is stopped, you can run :meth:`nni.experiment.Experiment.view` to restart web portal.
+
+.. tip::
+
+    This example uses :doc:`Python API </reference/experiment>` to create experiment.
+
+    You can also create and manage experiments with :doc:`command line tool <../hpo_nnictl/nnictl>`.
+
+
+.. rst-class:: sphx-glr-timing
+
+   **Total running time of the script:** ( 1 minutes  24.384 seconds)
+
+
+.. _sphx_glr_download_tutorials_hpo_quickstart_tensorflow_main.py:
+
+
+.. only :: html
+
+ .. container:: sphx-glr-footer
+    :class: sphx-glr-footer-example
+
+
+
+  .. container:: sphx-glr-download sphx-glr-download-python
+
+     :download:`Download Python source code: main.py <main.py>`
+
+
+
+  .. container:: sphx-glr-download sphx-glr-download-jupyter
+
+     :download:`Download Jupyter notebook: main.ipynb <main.ipynb>`
+
+
+.. only:: html
+
+ .. rst-class:: sphx-glr-signature
+
+    `Gallery generated by Sphinx-Gallery <https://sphinx-gallery.github.io>`_

docs/source/tutorials/nni_experiment.ipynb → docs/source/tutorials/hpo_quickstart_tensorflow/model.ipynb

@@ -15,14 +15,7 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "\n# Start and Manage a New Experiment\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "## Configure Search Space\n\n"
+        "\n# Port TensorFlow Quickstart to NNI\nThis is a modified version of `TensorFlow quickstart`_.\n\nIt can be run directly and will have the exact same result as original version.\n\nFurthermore, it enables the ability of auto tuning with an NNI *experiment*, which will be detailed later.\n\nIt is recommended to run this script directly first to verify the environment.\n\nThere are 3 key differences from the original version:\n\n1. In `Get optimized hyperparameters`_ part, it receives generated hyperparameters.\n2. In `(Optional) Report intermediate results`_ part, it reports per-epoch accuracy metrics.\n3. In `Report final result`_ part, it reports final accuracy.\n\n"
       ]
     },
     {
@@ -33,14 +26,14 @@
       },
       "outputs": [],
       "source": [
-        "search_space = {\n    \"C\": {\"_type\": \"quniform\", \"_value\": [0.1, 1, 0.1]},\n    \"kernel\": {\"_type\": \"choice\", \"_value\": [\"linear\", \"rbf\", \"poly\", \"sigmoid\"]},\n    \"degree\": {\"_type\": \"choice\", \"_value\": [1, 2, 3, 4]},\n    \"gamma\": {\"_type\": \"quniform\", \"_value\": [0.01, 0.1, 0.01]},\n    \"coef0\": {\"_type\": \"quniform\", \"_value\": [0.01, 0.1, 0.01]}\n}"
+        "import nni\nimport tensorflow as tf"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "## Configure Experiment\n\n"
+        "## Hyperparameters to be tuned\nThese are the hyperparameters that will be tuned later.\n\n"
       ]
     },
     {
@@ -51,14 +44,14 @@
       },
       "outputs": [],
       "source": [
-        "from nni.experiment import Experiment\nexperiment = Experiment('local')\nexperiment.config.experiment_name = 'Example'\nexperiment.config.trial_concurrency = 2\nexperiment.config.max_trial_number = 10\nexperiment.config.search_space = search_space\nexperiment.config.trial_command = 'python scripts/trial_sklearn.py'\nexperiment.config.trial_code_directory = './'\nexperiment.config.tuner.name = 'TPE'\nexperiment.config.tuner.class_args['optimize_mode'] = 'maximize'\nexperiment.config.training_service.use_active_gpu = True"
+        "params = {\n    'dense_units': 128,\n    'activation_type': 'relu',\n    'dropout_rate': 0.2,\n    'learning_rate': 0.001,\n}"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "## Start Experiment\n\n"
+        "## Get optimized hyperparameters\nIf run directly, :func:`nni.get_next_parameter` is a no-op and returns an empty dict.\nBut with an NNI *experiment*, it will receive optimized hyperparameters from tuning algorithm.\n\n"
       ]
     },
     {
@@ -69,14 +62,14 @@
       },
       "outputs": [],
       "source": [
-        "experiment.start(8080)"
+        "optimized_params = nni.get_next_parameter()\nparams.update(optimized_params)\nprint(params)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "## Experiment View & Control\n\nView the status of experiment.\n\n"
+        "## Load dataset\n\n"
       ]
     },
     {
@@ -87,14 +80,14 @@
       },
       "outputs": [],
       "source": [
-        "experiment.get_status()"
+        "mnist = tf.keras.datasets.mnist\n\n(x_train, y_train), (x_test, y_test) = mnist.load_data()\nx_train, x_test = x_train / 255.0, x_test / 255.0"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "Wait until at least one trial finishes.\n\n"
+        "## Build model with hyperparameters\n\n"
       ]
     },
     {
@@ -105,14 +98,14 @@
       },
       "outputs": [],
       "source": [
-        "import time\n\nfor _ in range(10):\n    stats = experiment.get_job_statistics()\n    if any(stat['trialJobStatus'] == 'SUCCEEDED' for stat in stats):\n        break\n    time.sleep(10)"
+        "model = tf.keras.models.Sequential([\n    tf.keras.layers.Flatten(input_shape=(28, 28)),\n    tf.keras.layers.Dense(params['dense_units'], activation=params['activation_type']),\n    tf.keras.layers.Dropout(params['dropout_rate']),\n    tf.keras.layers.Dense(10)\n])\n\nadam = tf.keras.optimizers.Adam(learning_rate=params['learning_rate'])\nloss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\nmodel.compile(optimizer=adam, loss=loss_fn, metrics=['accuracy'])"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "Export the experiment data.\n\n"
+        "## (Optional) Report intermediate results\nThe callback reports per-epoch accuracy to show learning curve in the web portal.\nYou can also leverage the metrics for early stopping with :doc:`NNI assessors </hpo/assessors>`.\n\nThis part can be safely skipped and the experiment will work fine.\n\n"
       ]
     },
     {
@@ -123,14 +116,14 @@
       },
       "outputs": [],
       "source": [
-        "experiment.export_data()"
+        "callback = tf.keras.callbacks.LambdaCallback(\n    on_epoch_end = lambda epoch, logs: nni.report_intermediate_result(logs['accuracy'])\n)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "Get metric of jobs\n\n"
+        "## Train and evluate the model\n\n"
       ]
     },
     {
@@ -141,14 +134,14 @@
       },
       "outputs": [],
       "source": [
-        "experiment.get_job_metrics()"
+        "model.fit(x_train, y_train, epochs=5, verbose=2, callbacks=[callback])\nloss, accuracy = model.evaluate(x_test, y_test, verbose=2)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "## Stop Experiment\n\n"
+        "## Report final result\nReport final accuracy to NNI so the tuning algorithm can suggest better hyperparameters.\n\n"
       ]
     },
     {
@@ -159,7 +152,7 @@
       },
       "outputs": [],
       "source": [
-        "experiment.stop()"
+        "nni.report_final_result(accuracy)"
       ]
     }
   ],
@@ -179,7 +172,7 @@
       "name": "python",
       "nbconvert_exporter": "python",
       "pygments_lexer": "ipython3",
-      "version": "3.8.8"
+      "version": "3.10.3"
     }
   },
   "nbformat": 4,

docs/source/tutorials/hpo_quickstart_tensorflow/model.py

+"""
+Port TensorFlow Quickstart to NNI
+=================================
+This is a modified version of `TensorFlow quickstart`_.
+
+It can be run directly and will have the exact same result as original version.
+
+Furthermore, it enables the ability of auto tuning with an NNI *experiment*, which will be detailed later.
+
+It is recommended to run this script directly first to verify the environment.
+
+There are 3 key differences from the original version:
+
+1. In `Get optimized hyperparameters`_ part, it receives generated hyperparameters.
+2. In `(Optional) Report intermediate results`_ part, it reports per-epoch accuracy metrics.
+3. In `Report final result`_ part, it reports final accuracy.
+
+.. _TensorFlow quickstart: https://www.tensorflow.org/tutorials/quickstart/beginner
+"""
+
+# %%
+import nni
+import tensorflow as tf
+
+# %%
+# Hyperparameters to be tuned
+# ---------------------------
+# These are the hyperparameters that will be tuned later.
+params = {
+    'dense_units': 128,
+    'activation_type': 'relu',
+    'dropout_rate': 0.2,
+    'learning_rate': 0.001,
+}
+
+# %%
+# Get optimized hyperparameters
+# -----------------------------
+# If run directly, :func:`nni.get_next_parameter` is a no-op and returns an empty dict.
+# But with an NNI *experiment*, it will receive optimized hyperparameters from tuning algorithm.
+optimized_params = nni.get_next_parameter()
+params.update(optimized_params)
+print(params)
+
+# %%
+# Load dataset
+# ------------
+mnist = tf.keras.datasets.mnist
+
+(x_train, y_train), (x_test, y_test) = mnist.load_data()
+x_train, x_test = x_train / 255.0, x_test / 255.0
+
+# %%
+# Build model with hyperparameters
+# --------------------------------
+model = tf.keras.models.Sequential([
+    tf.keras.layers.Flatten(input_shape=(28, 28)),
+    tf.keras.layers.Dense(params['dense_units'], activation=params['activation_type']),
+    tf.keras.layers.Dropout(params['dropout_rate']),
+    tf.keras.layers.Dense(10)
+])
+
+adam = tf.keras.optimizers.Adam(learning_rate=params['learning_rate'])
+loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+model.compile(optimizer=adam, loss=loss_fn, metrics=['accuracy'])
+
+# %%
+# (Optional) Report intermediate results
+# --------------------------------------
+# The callback reports per-epoch accuracy to show learning curve in the web portal.
+# You can also leverage the metrics for early stopping with :doc:`NNI assessors </hpo/assessors>`.
+#
+# This part can be safely skipped and the experiment will work fine.
+callback = tf.keras.callbacks.LambdaCallback(
+    on_epoch_end = lambda epoch, logs: nni.report_intermediate_result(logs['accuracy'])
+)
+
+# %%
+# Train and evluate the model
+# ---------------------------
+model.fit(x_train, y_train, epochs=5, verbose=2, callbacks=[callback])
+loss, accuracy = model.evaluate(x_test, y_test, verbose=2)
+
+# %%
+# Report final result
+# -------------------
+# Report final accuracy to NNI so the tuning algorithm can suggest better hyperparameters.
+nni.report_final_result(accuracy)

docs/source/tutorials/hpo_quickstart_tensorflow/model.py.md5

+d0c869d9d7c7f9208abc10357de52056
\ No newline at end of file

docs/source/tutorials/hpo_quickstart_tensorflow/model.rst

+:orphan:
+
+.. DO NOT EDIT.
+.. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY.
+.. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE:
+.. "tutorials/hpo_quickstart_tensorflow/model.py"
+.. LINE NUMBERS ARE GIVEN BELOW.
+
+.. only:: html
+
+    .. note::
+        :class: sphx-glr-download-link-note
+
+        Click :ref:`here <sphx_glr_download_tutorials_hpo_quickstart_tensorflow_model.py>`
+        to download the full example code
+
+.. rst-class:: sphx-glr-example-title
+
+.. _sphx_glr_tutorials_hpo_quickstart_tensorflow_model.py:
+
+
+Port TensorFlow Quickstart to NNI
+=================================
+This is a modified version of `TensorFlow quickstart`_.
+
+It can be run directly and will have the exact same result as original version.
+
+Furthermore, it enables the ability of auto tuning with an NNI *experiment*, which will be detailed later.
+
+It is recommended to run this script directly first to verify the environment.
+
+There are 3 key differences from the original version:
+
+1. In `Get optimized hyperparameters`_ part, it receives generated hyperparameters.
+2. In `(Optional) Report intermediate results`_ part, it reports per-epoch accuracy metrics.
+3. In `Report final result`_ part, it reports final accuracy.
+
+.. _TensorFlow quickstart: https://www.tensorflow.org/tutorials/quickstart/beginner
+
+.. GENERATED FROM PYTHON SOURCE LINES 22-25
+
+.. code-block:: default
+
+    import nni
+    import tensorflow as tf
+
+
+
+
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 26-29
+
+Hyperparameters to be tuned
+---------------------------
+These are the hyperparameters that will be tuned later.
+
+.. GENERATED FROM PYTHON SOURCE LINES 29-36
+
+.. code-block:: default
+
+    params = {
+        'dense_units': 128,
+        'activation_type': 'relu',
+        'dropout_rate': 0.2,
+        'learning_rate': 0.001,
+    }
+
+
+
+
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 37-41
+
+Get optimized hyperparameters
+-----------------------------
+If run directly, :func:`nni.get_next_parameter` is a no-op and returns an empty dict.
+But with an NNI *experiment*, it will receive optimized hyperparameters from tuning algorithm.
+
+.. GENERATED FROM PYTHON SOURCE LINES 41-45
+
+.. code-block:: default
+
+    optimized_params = nni.get_next_parameter()
+    params.update(optimized_params)
+    print(params)
+
+
+
+
+
+.. rst-class:: sphx-glr-script-out
+
+ Out:
+
+ .. code-block:: none
+
+    {'dense_units': 128, 'activation_type': 'relu', 'dropout_rate': 0.2, 'learning_rate': 0.001}
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 46-48
+
+Load dataset
+------------
+
+.. GENERATED FROM PYTHON SOURCE LINES 48-53
+
+.. code-block:: default
+
+    mnist = tf.keras.datasets.mnist
+
+    (x_train, y_train), (x_test, y_test) = mnist.load_data()
+    x_train, x_test = x_train / 255.0, x_test / 255.0
+
+
+
+
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 54-56
+
+Build model with hyperparameters
+--------------------------------
+
+.. GENERATED FROM PYTHON SOURCE LINES 56-67
+
+.. code-block:: default
+
+    model = tf.keras.models.Sequential([
+        tf.keras.layers.Flatten(input_shape=(28, 28)),
+        tf.keras.layers.Dense(params['dense_units'], activation=params['activation_type']),
+        tf.keras.layers.Dropout(params['dropout_rate']),
+        tf.keras.layers.Dense(10)
+    ])
+
+    adam = tf.keras.optimizers.Adam(learning_rate=params['learning_rate'])
+    loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+    model.compile(optimizer=adam, loss=loss_fn, metrics=['accuracy'])
+
+
+
+
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 68-74
+
+(Optional) Report intermediate results
+--------------------------------------
+The callback reports per-epoch accuracy to show learning curve in the web portal.
+You can also leverage the metrics for early stopping with :doc:`NNI assessors </hpo/assessors>`.
+
+This part can be safely skipped and the experiment will work fine.
+
+.. GENERATED FROM PYTHON SOURCE LINES 74-78
+
+.. code-block:: default
+
+    callback = tf.keras.callbacks.LambdaCallback(
+        on_epoch_end = lambda epoch, logs: nni.report_intermediate_result(logs['accuracy'])
+    )
+
+
+
+
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 79-81
+
+Train and evluate the model
+---------------------------
+
+.. GENERATED FROM PYTHON SOURCE LINES 81-84
+
+.. code-block:: default
+
+    model.fit(x_train, y_train, epochs=5, verbose=2, callbacks=[callback])
+    loss, accuracy = model.evaluate(x_test, y_test, verbose=2)
+
+
+
+
+
+.. rst-class:: sphx-glr-script-out
+
+ Out:
+
+ .. code-block:: none
+
+    Epoch 1/5
+    [2022-03-21 01:25:00] INFO (nni/MainThread) Intermediate result: 0.9153500199317932  (Index 0)
+    1875/1875 - 17s - loss: 0.2914 - accuracy: 0.9154 - 17s/epoch - 9ms/step
+    Epoch 2/5
+    [2022-03-21 01:25:18] INFO (nni/MainThread) Intermediate result: 0.9588666558265686  (Index 1)
+    1875/1875 - 18s - loss: 0.1387 - accuracy: 0.9589 - 18s/epoch - 10ms/step
+    Epoch 3/5
+    [2022-03-21 01:25:38] INFO (nni/MainThread) Intermediate result: 0.9677000045776367  (Index 2)
+    1875/1875 - 20s - loss: 0.1073 - accuracy: 0.9677 - 20s/epoch - 11ms/step
+    Epoch 4/5
+    [2022-03-21 01:25:56] INFO (nni/MainThread) Intermediate result: 0.9738666415214539  (Index 3)
+    1875/1875 - 18s - loss: 0.0866 - accuracy: 0.9739 - 18s/epoch - 10ms/step
+    Epoch 5/5
+    [2022-03-21 01:26:16] INFO (nni/MainThread) Intermediate result: 0.977483332157135  (Index 4)
+    1875/1875 - 21s - loss: 0.0728 - accuracy: 0.9775 - 21s/epoch - 11ms/step
+    313/313 - 2s - loss: 0.0702 - accuracy: 0.9776 - 2s/epoch - 6ms/step
+
+
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 85-88
+
+Report final result
+-------------------
+Report final accuracy to NNI so the tuning algorithm can suggest better hyperparameters.
+
+.. GENERATED FROM PYTHON SOURCE LINES 88-89
+
+.. code-block:: default
+
+    nni.report_final_result(accuracy)
+
+
+
+
+.. rst-class:: sphx-glr-script-out
+
+ Out:
+
+ .. code-block:: none
+
+    [2022-03-21 01:27:08] INFO (nni/MainThread) Final result: 0.9775999784469604
+
+
+
+
+
+.. rst-class:: sphx-glr-timing
+
+   **Total running time of the script:** ( 2 minutes  27.156 seconds)
+
+
+.. _sphx_glr_download_tutorials_hpo_quickstart_tensorflow_model.py:
+
+
+.. only :: html
+
+ .. container:: sphx-glr-footer
+    :class: sphx-glr-footer-example
+
+
+
+  .. container:: sphx-glr-download sphx-glr-download-python
+
+     :download:`Download Python source code: model.py <model.py>`
+
+
+
+  .. container:: sphx-glr-download sphx-glr-download-jupyter
+
+     :download:`Download Jupyter notebook: model.ipynb <model.ipynb>`
+
+
+.. only:: html
+
+ .. rst-class:: sphx-glr-signature
+
+    `Gallery generated by Sphinx-Gallery <https://sphinx-gallery.github.io>`_

docs/source/tutorials/hpo_quickstart_tensorflow/sg_execution_times.rst

+
+:orphan:
+
+.. _sphx_glr_tutorials_hpo_quickstart_tensorflow_sg_execution_times:
+
+Computation times
+=================
+**01:24.384** total execution time for **tutorials_hpo_quickstart_tensorflow** files:
+
++-----------------------------------------------------------------------------+-----------+--------+
+| :ref:`sphx_glr_tutorials_hpo_quickstart_tensorflow_main.py` (``main.py``)   | 01:24.384 | 0.0 MB |
++-----------------------------------------------------------------------------+-----------+--------+
+| :ref:`sphx_glr_tutorials_hpo_quickstart_tensorflow_model.py` (``model.py``) | 00:00.000 | 0.0 MB |
++-----------------------------------------------------------------------------+-----------+--------+