Merge pull request #2254 from microsoft/v1.5

Merge v1.5 branch back to master

examples/trials/mnist-nas/classic_mode/config_nas.yml

-authorName: default
-experimentName: example_mnist
-trialConcurrency: 1
-maxExecDuration: 1h
-maxTrialNum: 10
-#choice: local, remote, pai
-trainingServicePlatform: local
-#choice: true, false
-useAnnotation: true
-tuner:
-  codeDir: ../../../tuners/random_nas_tuner
-  classFileName: random_nas_tuner.py
-  className: RandomNASTuner
-trial:
-  command: python3 mnist.py
-  codeDir: .
-  gpuNum: 0
-  nasMode: classic_mode

examples/trials/mnist-nas/classic_mode/mnist.py

-"""A deep MNIST classifier using convolutional layers."""
-
-import argparse
-import logging
-import math
-import tempfile
-import time
-
-import tensorflow as tf
-from tensorflow.examples.tutorials.mnist import input_data
-
-import operators as op
-
-FLAGS = None
-
-logger = logging.getLogger('mnist_AutoML')
-
-
-class MnistNetwork(object):
-    '''
-    MnistNetwork is for initializing and building basic network for mnist.
-    '''
-    def __init__(self,
-                 channel_1_num,
-                 channel_2_num,
-                 conv_size,
-                 hidden_size,
-                 pool_size,
-                 learning_rate,
-                 x_dim=784,
-                 y_dim=10):
-        self.channel_1_num = channel_1_num
-        self.channel_2_num = channel_2_num
-        self.conv_size = conv_size
-        self.hidden_size = hidden_size
-        self.pool_size = pool_size
-        self.learning_rate = learning_rate
-        self.x_dim = x_dim
-        self.y_dim = y_dim
-
-        self.images = tf.placeholder(tf.float32, [None, self.x_dim], name='input_x')
-        self.labels = tf.placeholder(tf.float32, [None, self.y_dim], name='input_y')
-        self.keep_prob = tf.placeholder(tf.float32, name='keep_prob')
-
-        self.train_step = None
-        self.accuracy = None
-
-    def build_network(self):
-        '''
-        Building network for mnist, meanwhile specifying its neural architecture search space
-        '''
-
-        # Reshape to use within a convolutional neural net.
-        # Last dimension is for "features" - there is only one here, since images are
-        # grayscale -- it would be 3 for an RGB image, 4 for RGBA, etc.
-        with tf.name_scope('reshape'):
-            try:
-                input_dim = int(math.sqrt(self.x_dim))
-            except:
-                print(
-                    'input dim cannot be sqrt and reshape. input dim: ' + str(self.x_dim))
-                logger.debug(
-                    'input dim cannot be sqrt and reshape. input dim: %s', str(self.x_dim))
-                raise
-            x_image = tf.reshape(self.images, [-1, input_dim, input_dim, 1])
-
-        """@nni.mutable_layers(
-            {
-                layer_choice: [op.conv2d(size=1, in_ch=1, out_ch=self.channel_1_num),
-                               op.conv2d(size=3, in_ch=1, out_ch=self.channel_1_num),
-                               op.twice_conv2d(size=3, in_ch=1, out_ch=self.channel_1_num),
-                               op.twice_conv2d(size=7, in_ch=1, out_ch=self.channel_1_num),
-                               op.dilated_conv(in_ch=1, out_ch=self.channel_1_num),
-                               op.separable_conv(size=3, in_ch=1, out_ch=self.channel_1_num),
-                               op.separable_conv(size=5, in_ch=1, out_ch=self.channel_1_num),
-                               op.separable_conv(size=7, in_ch=1, out_ch=self.channel_1_num)],
-                fixed_inputs: [x_image],
-                layer_output: conv1_out
-            },
-            {
-                layer_choice: [op.post_process(ch_size=self.channel_1_num)],
-                fixed_inputs: [conv1_out],
-                layer_output: post1_out
-            },
-            {
-                layer_choice: [op.max_pool(size=3),
-                               op.max_pool(size=5),
-                               op.max_pool(size=7),
-                               op.avg_pool(size=3),
-                               op.avg_pool(size=5),
-                               op.avg_pool(size=7)],
-                fixed_inputs: [post1_out],
-                layer_output: pool1_out
-            },
-            {
-                layer_choice: [op.conv2d(size=1, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.conv2d(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.twice_conv2d(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.twice_conv2d(size=7, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.dilated_conv(in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.separable_conv(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.separable_conv(size=5, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.separable_conv(size=7, in_ch=self.channel_1_num, out_ch=self.channel_2_num)],
-                fixed_inputs: [pool1_out],
-                optional_inputs: [post1_out],
-                optional_input_size: [0, 1],
-                layer_output: conv2_out
-            },
-            {
-                layer_choice: [op.post_process(ch_size=self.channel_2_num)],
-                fixed_inputs: [conv2_out],
-                layer_output: post2_out
-            },
-            {
-                layer_choice: [op.max_pool(size=3),
-                               op.max_pool(size=5),
-                               op.max_pool(size=7),
-                               op.avg_pool(size=3),
-                               op.avg_pool(size=5),
-                               op.avg_pool(size=7)],
-                fixed_inputs: [post2_out],
-                optional_inputs: [post1_out, pool1_out],
-                optional_input_size: [0, 1],
-                layer_output: pool2_out
-            }
-        )"""
-
-        # Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image
-        # is down to 7x7x64 feature maps -- maps this to 1024 features.
-        last_dim_list = pool2_out.get_shape().as_list()
-        assert(last_dim_list[1] == last_dim_list[2])
-        last_dim = last_dim_list[1]
-        with tf.name_scope('fc1'):
-            w_fc1 = op.weight_variable(
-                [last_dim * last_dim * self.channel_2_num, self.hidden_size])
-            b_fc1 = op.bias_variable([self.hidden_size])
-
-        h_pool2_flat = tf.reshape(
-            pool2_out, [-1, last_dim * last_dim * self.channel_2_num])
-        h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, w_fc1) + b_fc1)
-
-        # Dropout - controls the complexity of the model, prevents co-adaptation of features.
-        with tf.name_scope('dropout'):
-            h_fc1_drop = tf.nn.dropout(h_fc1, self.keep_prob)
-
-        # Map the 1024 features to 10 classes, one for each digit
-        with tf.name_scope('fc2'):
-            w_fc2 = op.weight_variable([self.hidden_size, self.y_dim])
-            b_fc2 = op.bias_variable([self.y_dim])
-            y_conv = tf.matmul(h_fc1_drop, w_fc2) + b_fc2
-
-        with tf.name_scope('loss'):
-            cross_entropy = tf.reduce_mean(
-                tf.nn.softmax_cross_entropy_with_logits(labels=self.labels, logits=y_conv))
-        with tf.name_scope('adam_optimizer'):
-            self.train_step = tf.train.AdamOptimizer(
-                self.learning_rate).minimize(cross_entropy)
-
-        with tf.name_scope('accuracy'):
-            correct_prediction = tf.equal(
-                tf.argmax(y_conv, 1), tf.argmax(self.labels, 1))
-            self.accuracy = tf.reduce_mean(
-                tf.cast(correct_prediction, tf.float32))
-
-
-def download_mnist_retry(data_dir, max_num_retries=20):
-    """Try to download mnist dataset and avoid errors"""
-    for _ in range(max_num_retries):
-        try:
-            return input_data.read_data_sets(data_dir, one_hot=True)
-        except tf.errors.AlreadyExistsError:
-            time.sleep(1)
-    raise Exception("Failed to download MNIST.")
-
-def main(params):
-    '''
-    Main function, build mnist network, run and send result to NNI.
-    '''
-    # Import data
-    mnist = download_mnist_retry(params['data_dir'])
-    print('Mnist download data done.')
-    logger.debug('Mnist download data done.')
-
-    # Create the model
-    # Build the graph for the deep net
-    mnist_network = MnistNetwork(channel_1_num=params['channel_1_num'],
-                                 channel_2_num=params['channel_2_num'],
-                                 conv_size=params['conv_size'],
-                                 hidden_size=params['hidden_size'],
-                                 pool_size=params['pool_size'],
-                                 learning_rate=params['learning_rate'])
-    mnist_network.build_network()
-    logger.debug('Mnist build network done.')
-
-    # Write log
-    graph_location = tempfile.mkdtemp()
-    logger.debug('Saving graph to: %s', graph_location)
-    train_writer = tf.summary.FileWriter(graph_location)
-    train_writer.add_graph(tf.get_default_graph())
-
-    test_acc = 0.0
-    with tf.Session() as sess:
-        sess.run(tf.global_variables_initializer())
-        for i in range(params['batch_num']):
-            batch = mnist.train.next_batch(params['batch_size'])
-            mnist_network.train_step.run(feed_dict={mnist_network.images: batch[0],
-                                                    mnist_network.labels: batch[1],
-                                                    mnist_network.keep_prob: 1 - params['dropout_rate']}
-                                        )
-
-            if i % 100 == 0:
-                test_acc = mnist_network.accuracy.eval(
-                    feed_dict={mnist_network.images: mnist.test.images,
-                               mnist_network.labels: mnist.test.labels,
-                               mnist_network.keep_prob: 1.0})
-
-                """@nni.report_intermediate_result(test_acc)"""
-                logger.debug('test accuracy %g', test_acc)
-                logger.debug('Pipe send intermediate result done.')
-
-        test_acc = mnist_network.accuracy.eval(
-            feed_dict={mnist_network.images: mnist.test.images,
-                       mnist_network.labels: mnist.test.labels,
-                       mnist_network.keep_prob: 1.0})
-
-        """@nni.report_final_result(test_acc)"""
-        logger.debug('Final result is %g', test_acc)
-        logger.debug('Send final result done.')
-
-def get_params():
-    ''' Get parameters from command line '''
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--data_dir", type=str, default='/tmp/tensorflow/mnist/input_data', help="data directory")
-    parser.add_argument("--dropout_rate", type=float, default=0.5, help="dropout rate")
-    parser.add_argument("--channel_1_num", type=int, default=32)
-    parser.add_argument("--channel_2_num", type=int, default=64)
-    parser.add_argument("--conv_size", type=int, default=5)
-    parser.add_argument("--pool_size", type=int, default=2)
-    parser.add_argument("--hidden_size", type=int, default=1024)
-    parser.add_argument("--learning_rate", type=float, default=1e-4)
-    parser.add_argument("--batch_num", type=int, default=2000)
-    parser.add_argument("--batch_size", type=int, default=32)
-
-    args, _ = parser.parse_known_args()
-    return args
-
-if __name__ == '__main__':
-    try:
-        params = vars(get_params())
-        main(params)
-    except Exception as exception:
-        logger.exception(exception)
-        raise

examples/trials/mnist-nas/classic_mode/operators.py

-import tensorflow as tf
-import math
-
-
-def weight_variable(shape):
-    """weight_variable generates a weight variable of a given shape."""
-    initial = tf.truncated_normal(shape, stddev=0.1)
-    return tf.Variable(initial)
-
-def bias_variable(shape):
-    """bias_variable generates a bias variable of a given shape."""
-    initial = tf.constant(0.1, shape=shape)
-    return tf.Variable(initial)
-
-def sum_op(inputs):
-    """sum_op"""
-    fixed_input = inputs[0][0]
-    optional_input = inputs[1][0]
-    fixed_shape = fixed_input.get_shape().as_list()
-    optional_shape = optional_input.get_shape().as_list()
-    assert fixed_shape[1] == fixed_shape[2]
-    assert optional_shape[1] == optional_shape[2]
-    pool_size = math.ceil(optional_shape[1] / fixed_shape[1])
-    pool_out = tf.nn.avg_pool(optional_input, ksize=[1, pool_size, pool_size, 1], strides=[1, pool_size, pool_size, 1], padding='SAME')
-    conv_matrix = weight_variable([1, 1, optional_shape[3], fixed_shape[3]])
-    conv_out = tf.nn.conv2d(pool_out, conv_matrix, strides=[1, 1, 1, 1], padding='SAME')
-    return fixed_input + conv_out
-
-
-def conv2d(inputs, size=-1, in_ch=-1, out_ch=-1):
-    """conv2d returns a 2d convolution layer with full stride."""
-    if not inputs[1]:
-        x_input = inputs[0][0]
-    else:
-        x_input = sum_op(inputs)
-    if size in [1, 3]:
-        w_matrix = weight_variable([size, size, in_ch, out_ch])
-        return tf.nn.conv2d(x_input, w_matrix, strides=[1, 1, 1, 1], padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-def twice_conv2d(inputs, size=-1, in_ch=-1, out_ch=-1):
-    """twice_conv2d"""
-    if not inputs[1]:
-        x_input = inputs[0][0]
-    else:
-        x_input = sum_op(inputs)
-    if size in  [3, 7]:
-        w_matrix1 = weight_variable([1, size, in_ch, int(out_ch/2)])
-        out = tf.nn.conv2d(x_input, w_matrix1, strides=[1, 1, 1, 1], padding='SAME')
-        w_matrix2 = weight_variable([size, 1, int(out_ch/2), out_ch])
-        return tf.nn.conv2d(out, w_matrix2, strides=[1, 1, 1, 1], padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-def dilated_conv(inputs, size=3, in_ch=-1, out_ch=-1):
-    """dilated_conv"""
-    if not inputs[1]:
-        x_input = inputs[0][0]
-    else:
-        x_input = sum_op(inputs)
-    if size == 3:
-        w_matrix = weight_variable([size, size, in_ch, out_ch])
-        return tf.nn.atrous_conv2d(x_input, w_matrix, rate=2, padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-def separable_conv(inputs, size=-1, in_ch=-1, out_ch=-1):
-    """separable_conv"""
-    if not inputs[1]:
-        x_input = inputs[0][0]
-    else:
-        x_input = sum_op(inputs)
-    if size in [3, 5, 7]:
-        depth_matrix = weight_variable([size, size, in_ch, 1])
-        point_matrix = weight_variable([1, 1, 1*in_ch, out_ch])
-        return tf.nn.separable_conv2d(x_input, depth_matrix, point_matrix, strides=[1, 1, 1, 1], padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-
-def avg_pool(inputs, size=-1):
-    """avg_pool downsamples a feature map."""
-    if not inputs[1]:
-        x_input = inputs[0][0]
-    else:
-        x_input = sum_op(inputs)
-    if size in [3, 5, 7]:
-        return tf.nn.avg_pool(x_input, ksize=[1, size, size, 1], strides=[1, size, size, 1], padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-def max_pool(inputs, size=-1):
-    """max_pool downsamples a feature map."""
-    if not inputs[1]:
-        x_input = inputs[0][0]
-    else:
-        x_input = sum_op(inputs)
-    if size in [3, 5, 7]:
-        return tf.nn.max_pool(x_input, ksize=[1, size, size, 1], strides=[1, size, size, 1], padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-
-def post_process(inputs, ch_size=-1):
-    """post_process"""
-    x_input = inputs[0][0]
-    bias_matrix = bias_variable([ch_size])
-    return tf.nn.relu(x_input + bias_matrix)

examples/trials/mnist-nas/config_ppo.yml

-authorName: NNI-example
-experimentName: example_mnist
-trialConcurrency: 1
-maxExecDuration: 100h
-maxTrialNum: 10000
-#choice: local, remote, pai
-trainingServicePlatform: local
-#choice: true, false
-useAnnotation: true
-tuner:
-  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner
-  #SMAC, PPO (SMAC and PPO should be installed through nnictl)
-  builtinTunerName: PPOTuner
-  classArgs:
-    optimize_mode: maximize
-trial:
-  command: python3 mnist.py
-  codeDir: .
-  gpuNum: 0

examples/trials/mnist-nas/darts_mode/config_darts.yml

-authorName: default
-experimentName: example_mnist
-trialConcurrency: 1
-maxExecDuration: 1h
-maxTrialNum: 10
-#choice: local, remote, pai
-trainingServicePlatform: local
-#choice: true, false
-useAnnotation: true
-tuner:
-  codeDir: ../../../tuners/random_nas_tuner
-  classFileName: random_nas_tuner.py
-  className: RandomNASTuner
-trial:
-  command: python3 mnist-darts.py
-  codeDir: .
-  gpuNum: 0
-  nasMode: oneshot_mode

examples/trials/mnist-nas/darts_mode/mnist-darts.py

-"""A deep MNIST classifier using convolutional layers."""
-
-import argparse
-import logging
-import math
-import tempfile
-import time
-
-import tensorflow as tf
-from tensorflow.examples.tutorials.mnist import input_data
-
-import operators as op
-
-FLAGS = None
-
-logger = logging.getLogger('mnist_AutoML')
-
-
-class MnistNetwork(object):
-    '''
-    MnistNetwork is for initializing and building basic network for mnist.
-    '''
-    def __init__(self,
-                 channel_1_num,
-                 channel_2_num,
-                 conv_size,
-                 hidden_size,
-                 pool_size,
-                 learning_rate,
-                 x_dim=784,
-                 y_dim=10):
-        self.channel_1_num = channel_1_num
-        self.channel_2_num = channel_2_num
-        self.conv_size = conv_size
-        self.hidden_size = hidden_size
-        self.pool_size = pool_size
-        self.learning_rate = learning_rate
-        self.x_dim = x_dim
-        self.y_dim = y_dim
-
-        self.images = tf.placeholder(tf.float32, [None, self.x_dim], name='input_x')
-        self.labels = tf.placeholder(tf.float32, [None, self.y_dim], name='input_y')
-        self.keep_prob = tf.placeholder(tf.float32, name='keep_prob')
-
-        self.train_step = None
-        self.accuracy = None
-
-    def build_network(self):
-        '''
-        Building network for mnist, meanwhile specifying its neural architecture search space
-        '''
-
-        # Reshape to use within a convolutional neural net.
-        # Last dimension is for "features" - there is only one here, since images are
-        # grayscale -- it would be 3 for an RGB image, 4 for RGBA, etc.
-        with tf.name_scope('reshape'):
-            try:
-                input_dim = int(math.sqrt(self.x_dim))
-            except:
-                print(
-                    'input dim cannot be sqrt and reshape. input dim: ' + str(self.x_dim))
-                logger.debug(
-                    'input dim cannot be sqrt and reshape. input dim: %s', str(self.x_dim))
-                raise
-            x_image = tf.reshape(self.images, [-1, input_dim, input_dim, 1])
-
-        """@nni.mutable_layers(
-            {
-                layer_choice: [op.conv2d(size=1, in_ch=1, out_ch=self.channel_1_num),
-                               op.conv2d(size=3, in_ch=1, out_ch=self.channel_1_num),
-                               op.twice_conv2d(size=3, in_ch=1, out_ch=self.channel_1_num),
-                               op.twice_conv2d(size=7, in_ch=1, out_ch=self.channel_1_num),
-                               op.dilated_conv(in_ch=1, out_ch=self.channel_1_num),
-                               op.separable_conv(size=3, in_ch=1, out_ch=self.channel_1_num),
-                               op.separable_conv(size=5, in_ch=1, out_ch=self.channel_1_num),
-                               op.separable_conv(size=7, in_ch=1, out_ch=self.channel_1_num)],
-                fixed_inputs: [x_image],
-                layer_output: conv1_out
-            },
-            {
-                layer_choice: [op.post_process(ch_size=self.channel_1_num)],
-                fixed_inputs: [conv1_out],
-                layer_output: post1_out
-            },
-            {
-                layer_choice: [op.max_pool(size=3),
-                               op.max_pool(size=5),
-                               op.max_pool(size=7),
-                               op.avg_pool(size=3),
-                               op.avg_pool(size=5),
-                               op.avg_pool(size=7)],
-                fixed_inputs: [post1_out],
-                layer_output: pool1_out
-            },
-            {
-                layer_choice: [op.conv2d(size=1, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.conv2d(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.twice_conv2d(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.twice_conv2d(size=7, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.dilated_conv(in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.separable_conv(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.separable_conv(size=5, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.separable_conv(size=7, in_ch=self.channel_1_num, out_ch=self.channel_2_num)],
-                fixed_inputs: [pool1_out],
-                optional_inputs: [post1_out],
-                optional_input_size: [0, 1],
-                layer_output: conv2_out
-            },
-            {
-                layer_choice: [op.post_process(ch_size=self.channel_2_num)],
-                fixed_inputs: [conv2_out],
-                layer_output: post2_out
-            },
-            {
-                layer_choice: [op.max_pool(size=3),
-                               op.max_pool(size=5),
-                               op.max_pool(size=7),
-                               op.avg_pool(size=3),
-                               op.avg_pool(size=5),
-                               op.avg_pool(size=7)],
-                fixed_inputs: [post2_out],
-                optional_inputs: [post1_out, pool1_out],
-                optional_input_size: [0, 1],
-                layer_output: pool2_out
-            }
-        )"""
-
-        # Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image
-        # is down to 7x7x64 feature maps -- maps this to 1024 features.
-        last_dim_list = pool2_out.get_shape().as_list()
-        assert(last_dim_list[1] == last_dim_list[2])
-        last_dim = last_dim_list[1]
-        with tf.name_scope('fc1'):
-            w_fc1 = op.weight_variable(
-                [last_dim * last_dim * self.channel_2_num, self.hidden_size])
-            b_fc1 = op.bias_variable([self.hidden_size])
-
-        h_pool2_flat = tf.reshape(
-            pool2_out, [-1, last_dim * last_dim * self.channel_2_num])
-        h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, w_fc1) + b_fc1)
-
-        # Dropout - controls the complexity of the model, prevents co-adaptation of features.
-        with tf.name_scope('dropout'):
-            h_fc1_drop = tf.nn.dropout(h_fc1, self.keep_prob)
-
-        # Map the 1024 features to 10 classes, one for each digit
-        with tf.name_scope('fc2'):
-            w_fc2 = op.weight_variable([self.hidden_size, self.y_dim])
-            b_fc2 = op.bias_variable([self.y_dim])
-            y_conv = tf.matmul(h_fc1_drop, w_fc2) + b_fc2
-
-        with tf.name_scope('loss'):
-            self.cross_entropy = tf.reduce_mean(
-                tf.nn.softmax_cross_entropy_with_logits(labels=self.labels, logits=y_conv))
-        with tf.name_scope('adam_optimizer'):
-            self.train_step = tf.train.AdamOptimizer(
-                self.learning_rate).minimize(self.cross_entropy)
-
-        with tf.name_scope('accuracy'):
-            correct_prediction = tf.equal(
-                tf.argmax(y_conv, 1), tf.argmax(self.labels, 1))
-            self.accuracy = tf.reduce_mean(
-                tf.cast(correct_prediction, tf.float32))
-
-
-def download_mnist_retry(data_dir, max_num_retries=20):
-    """Try to download mnist dataset and avoid errors"""
-    for _ in range(max_num_retries):
-        try:
-            return input_data.read_data_sets(data_dir, one_hot=True)
-        except tf.errors.AlreadyExistsError:
-            time.sleep(1)
-    raise Exception("Failed to download MNIST.")
-
-def main(params):
-    '''
-    Main function, build mnist network, run and send result to NNI.
-    '''
-    # Import data
-    mnist = download_mnist_retry(params['data_dir'])
-    print('Mnist download data done.')
-    logger.debug('Mnist download data done.')
-
-    # Create the model
-    # Build the graph for the deep net
-    mnist_network = MnistNetwork(channel_1_num=params['channel_1_num'],
-                                 channel_2_num=params['channel_2_num'],
-                                 conv_size=params['conv_size'],
-                                 hidden_size=params['hidden_size'],
-                                 pool_size=params['pool_size'],
-                                 learning_rate=params['learning_rate'])
-    mnist_network.build_network()
-    logger.debug('Mnist build network done.')
-
-    # Write log
-    graph_location = tempfile.mkdtemp()
-    logger.debug('Saving graph to: %s', graph_location)
-    train_writer = tf.summary.FileWriter(graph_location)
-    train_writer.add_graph(tf.get_default_graph())
-
-    test_acc = 0.0
-    with tf.Session() as sess:
-        sess.run(tf.global_variables_initializer())
-        for i in range(params['batch_num']):
-            batch = mnist.train.next_batch(params['batch_size'])
-            feed_dict={mnist_network.images: batch[0],
-                        mnist_network.labels: batch[1],
-                        mnist_network.keep_prob: 1 - params['dropout_rate']}
-            """@nni.training_update(tf, sess, mnist_network.cross_entropy)"""
-            batch = mnist.train.next_batch(params['batch_size'])
-            feed_dict={mnist_network.images: batch[0],
-                        mnist_network.labels: batch[1],
-                        mnist_network.keep_prob: 1 - params['dropout_rate']}
-            mnist_network.train_step.run(feed_dict=feed_dict)
-
-            if i % 100 == 0:
-                test_acc = mnist_network.accuracy.eval(
-                    feed_dict={mnist_network.images: mnist.test.images,
-                               mnist_network.labels: mnist.test.labels,
-                               mnist_network.keep_prob: 1.0})
-
-                """@nni.report_intermediate_result(test_acc)"""
-                logger.debug('test accuracy %g', test_acc)
-                logger.debug('Pipe send intermediate result done.')
-
-        test_acc = mnist_network.accuracy.eval(
-            feed_dict={mnist_network.images: mnist.test.images,
-                       mnist_network.labels: mnist.test.labels,
-                       mnist_network.keep_prob: 1.0})
-
-        """@nni.report_final_result(test_acc)"""
-        logger.debug('Final result is %g', test_acc)
-        logger.debug('Send final result done.')
-
-def get_params():
-    ''' Get parameters from command line '''
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--data_dir", type=str, default='/tmp/tensorflow/mnist/input_data', help="data directory")
-    parser.add_argument("--dropout_rate", type=float, default=0.5, help="dropout rate")
-    parser.add_argument("--channel_1_num", type=int, default=32)
-    parser.add_argument("--channel_2_num", type=int, default=64)
-    parser.add_argument("--conv_size", type=int, default=5)
-    parser.add_argument("--pool_size", type=int, default=2)
-    parser.add_argument("--hidden_size", type=int, default=1024)
-    parser.add_argument("--learning_rate", type=float, default=1e-4)
-    parser.add_argument("--batch_num", type=int, default=2000)
-    parser.add_argument("--batch_size", type=int, default=32)
-
-    args, _ = parser.parse_known_args()
-    return args
-
-if __name__ == '__main__':
-    try:
-        params = vars(get_params())
-        main(params)
-    except Exception as exception:
-        logger.exception(exception)
-        raise

examples/trials/mnist-nas/darts_mode/operators.py

-import tensorflow as tf
-import math
-
-
-def weight_variable(shape):
-    """weight_variable generates a weight variable of a given shape."""
-    initial = tf.truncated_normal(shape, stddev=0.1)
-    return tf.Variable(initial)
-
-def bias_variable(shape):
-    """bias_variable generates a bias variable of a given shape."""
-    initial = tf.constant(0.1, shape=shape)
-    return tf.Variable(initial)
-
-def sum_op(inputs):
-    """sum_op"""
-    fixed_input = inputs[0][0]
-    optional_input = tf.concat(inputs[1], axis=3)
-    fixed_shape = fixed_input.get_shape().as_list()
-    optional_shape = optional_input.get_shape().as_list()
-    assert fixed_shape[1] == fixed_shape[2]
-    assert optional_shape[1] == optional_shape[2]
-    pool_size = math.ceil(optional_shape[1] / fixed_shape[1])
-    pool_out = tf.nn.avg_pool(optional_input, ksize=[1, pool_size, pool_size, 1], strides=[1, pool_size, pool_size, 1], padding='SAME')
-    conv_matrix = weight_variable([1, 1, optional_shape[3], fixed_shape[3]])
-    conv_out = tf.nn.conv2d(pool_out, conv_matrix, strides=[1, 1, 1, 1], padding='SAME')
-    return fixed_input + conv_out
-
-
-def conv2d(inputs, size=-1, in_ch=-1, out_ch=-1):
-    """conv2d returns a 2d convolution layer with full stride."""
-    if not inputs[1]:
-        x_input = inputs[0][0]
-    else:
-        x_input = sum_op(inputs)
-    if size in [1, 3]:
-        w_matrix = weight_variable([size, size, in_ch, out_ch])
-        return tf.nn.conv2d(x_input, w_matrix, strides=[1, 1, 1, 1], padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-def twice_conv2d(inputs, size=-1, in_ch=-1, out_ch=-1):
-    """twice_conv2d"""
-    if not inputs[1]:
-        x_input = inputs[0][0]
-    else:
-        x_input = sum_op(inputs)
-    if size in  [3, 7]:
-        w_matrix1 = weight_variable([1, size, in_ch, int(out_ch/2)])
-        out = tf.nn.conv2d(x_input, w_matrix1, strides=[1, 1, 1, 1], padding='SAME')
-        w_matrix2 = weight_variable([size, 1, int(out_ch/2), out_ch])
-        return tf.nn.conv2d(out, w_matrix2, strides=[1, 1, 1, 1], padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-def dilated_conv(inputs, size=3, in_ch=-1, out_ch=-1):
-    """dilated_conv"""
-    if not inputs[1]:
-        x_input = inputs[0][0]
-    else:
-        x_input = sum_op(inputs)
-    if size == 3:
-        w_matrix = weight_variable([size, size, in_ch, out_ch])
-        return tf.nn.atrous_conv2d(x_input, w_matrix, rate=2, padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-def separable_conv(inputs, size=-1, in_ch=-1, out_ch=-1):
-    """separable_conv"""
-    if not inputs[1]:
-        x_input = inputs[0][0]
-    else:
-        x_input = sum_op(inputs)
-    if size in [3, 5, 7]:
-        depth_matrix = weight_variable([size, size, in_ch, 1])
-        point_matrix = weight_variable([1, 1, 1*in_ch, out_ch])
-        return tf.nn.separable_conv2d(x_input, depth_matrix, point_matrix, strides=[1, 1, 1, 1], padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-
-def avg_pool(inputs, size=-1):
-    """avg_pool downsamples a feature map."""
-    if not inputs[1]:
-        x_input = inputs[0][0]
-    else:
-        x_input = sum_op(inputs)
-    if size in [3, 5, 7]:
-        return tf.nn.avg_pool(x_input, ksize=[1, size, size, 1], strides=[1, 1, 1, 1], padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-def max_pool(inputs, size=-1):
-    """max_pool downsamples a feature map."""
-    if not inputs[1]:
-        x_input = inputs[0][0]
-    else:
-        x_input = sum_op(inputs)
-    if size in [3, 5, 7]:
-        return tf.nn.max_pool(x_input, ksize=[1, size, size, 1], strides=[1, 1, 1, 1], padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-
-def post_process(inputs, ch_size=-1):
-    """post_process"""
-    x_input = inputs[0][0]
-    bias_matrix = bias_variable([ch_size])
-    return tf.nn.relu(x_input + bias_matrix)

examples/trials/mnist-nas/enas_mode/config_enas.yml

-authorName: default
-experimentName: example_mnist
-trialConcurrency: 1
-maxExecDuration: 1h
-maxTrialNum: 10
-#choice: local, remote, pai
-trainingServicePlatform: local
-#choice: true, false
-useAnnotation: true
-multiPhase: true
-tuner:
-  codeDir: ../../../tuners/random_nas_tuner
-  classFileName: random_nas_tuner.py
-  className: RandomNASTuner
-trial:
-  command: python3 mnist-enas.py
-  codeDir: .
-  gpuNum: 0
-  nasMode: enas_mode

examples/trials/mnist-nas/enas_mode/mnist-enas.py

-"""A deep MNIST classifier using convolutional layers."""
-
-import argparse
-import logging
-import math
-import tempfile
-import time
-
-import tensorflow as tf
-from tensorflow.examples.tutorials.mnist import input_data
-
-import operators as op
-
-FLAGS = None
-
-logger = logging.getLogger('mnist_AutoML')
-
-
-class MnistNetwork(object):
-    '''
-    MnistNetwork is for initializing and building basic network for mnist.
-    '''
-    def __init__(self,
-                 channel_1_num,
-                 channel_2_num,
-                 conv_size,
-                 hidden_size,
-                 pool_size,
-                 learning_rate,
-                 x_dim=784,
-                 y_dim=10):
-        self.channel_1_num = channel_1_num
-        self.channel_2_num = channel_2_num
-        self.conv_size = conv_size
-        self.hidden_size = hidden_size
-        self.pool_size = pool_size
-        self.learning_rate = learning_rate
-        self.x_dim = x_dim
-        self.y_dim = y_dim
-
-        self.images = tf.placeholder(tf.float32, [None, self.x_dim], name='input_x')
-        self.labels = tf.placeholder(tf.float32, [None, self.y_dim], name='input_y')
-        self.keep_prob = tf.placeholder(tf.float32, name='keep_prob')
-
-        self.train_step = None
-        self.accuracy = None
-
-    def build_network(self):
-        '''
-        Building network for mnist, meanwhile specifying its neural architecture search space
-        '''
-
-        # Reshape to use within a convolutional neural net.
-        # Last dimension is for "features" - there is only one here, since images are
-        # grayscale -- it would be 3 for an RGB image, 4 for RGBA, etc.
-        with tf.name_scope('reshape'):
-            try:
-                input_dim = int(math.sqrt(self.x_dim))
-            except:
-                print(
-                    'input dim cannot be sqrt and reshape. input dim: ' + str(self.x_dim))
-                logger.debug(
-                    'input dim cannot be sqrt and reshape. input dim: %s', str(self.x_dim))
-                raise
-            x_image = tf.reshape(self.images, [-1, input_dim, input_dim, 1])
-
-        """@nni.mutable_layers(
-            {
-                layer_choice: [op.conv2d(size=1, in_ch=1, out_ch=self.channel_1_num),
-                               op.conv2d(size=3, in_ch=1, out_ch=self.channel_1_num),
-                               op.twice_conv2d(size=3, in_ch=1, out_ch=self.channel_1_num),
-                               op.twice_conv2d(size=7, in_ch=1, out_ch=self.channel_1_num),
-                               op.dilated_conv(in_ch=1, out_ch=self.channel_1_num),
-                               op.separable_conv(size=3, in_ch=1, out_ch=self.channel_1_num),
-                               op.separable_conv(size=5, in_ch=1, out_ch=self.channel_1_num),
-                               op.separable_conv(size=7, in_ch=1, out_ch=self.channel_1_num)],
-                fixed_inputs: [x_image],
-                layer_output: conv1_out
-            },
-            {
-                layer_choice: [op.post_process(ch_size=self.channel_1_num)],
-                fixed_inputs: [conv1_out],
-                layer_output: post1_out
-            },
-            {
-                layer_choice: [op.max_pool(size=3),
-                               op.max_pool(size=5),
-                               op.max_pool(size=7),
-                               op.avg_pool(size=3),
-                               op.avg_pool(size=5),
-                               op.avg_pool(size=7)],
-                fixed_inputs: [post1_out],
-                layer_output: pool1_out
-            },
-            {
-                layer_choice: [op.conv2d(size=1, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.conv2d(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.twice_conv2d(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.twice_conv2d(size=7, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.dilated_conv(in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.separable_conv(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.separable_conv(size=5, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.separable_conv(size=7, in_ch=self.channel_1_num, out_ch=self.channel_2_num)],
-                fixed_inputs: [pool1_out],
-                optional_inputs: [post1_out],
-                optional_input_size: [0, 1],
-                layer_output: conv2_out
-            },
-            {
-                layer_choice: [op.post_process(ch_size=self.channel_2_num)],
-                fixed_inputs: [conv2_out],
-                layer_output: post2_out
-            },
-            {
-                layer_choice: [op.max_pool(size=3),
-                               op.max_pool(size=5),
-                               op.max_pool(size=7),
-                               op.avg_pool(size=3),
-                               op.avg_pool(size=5),
-                               op.avg_pool(size=7)],
-                fixed_inputs: [post2_out],
-                optional_inputs: [post1_out, pool1_out],
-                optional_input_size: [0, 1],
-                layer_output: pool2_out
-            }
-        )"""
-
-        # Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image
-        # is down to 7x7x64 feature maps -- maps this to 1024 features.
-        last_dim_list = pool2_out.get_shape().as_list()
-        assert(last_dim_list[1] == last_dim_list[2])
-        last_dim = last_dim_list[1]
-        with tf.name_scope('fc1'):
-            w_fc1 = op.weight_variable(
-                [last_dim * last_dim * self.channel_2_num, self.hidden_size])
-            b_fc1 = op.bias_variable([self.hidden_size])
-
-        h_pool2_flat = tf.reshape(
-            pool2_out, [-1, last_dim * last_dim * self.channel_2_num])
-        h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, w_fc1) + b_fc1)
-
-        # Dropout - controls the complexity of the model, prevents co-adaptation of features.
-        with tf.name_scope('dropout'):
-            h_fc1_drop = tf.nn.dropout(h_fc1, self.keep_prob)
-
-        # Map the 1024 features to 10 classes, one for each digit
-        with tf.name_scope('fc2'):
-            w_fc2 = op.weight_variable([self.hidden_size, self.y_dim])
-            b_fc2 = op.bias_variable([self.y_dim])
-            y_conv = tf.matmul(h_fc1_drop, w_fc2) + b_fc2
-
-        with tf.name_scope('loss'):
-            cross_entropy = tf.reduce_mean(
-                tf.nn.softmax_cross_entropy_with_logits(labels=self.labels, logits=y_conv))
-        with tf.name_scope('adam_optimizer'):
-            self.train_step = tf.train.AdamOptimizer(
-                self.learning_rate).minimize(cross_entropy)
-
-        with tf.name_scope('accuracy'):
-            correct_prediction = tf.equal(
-                tf.argmax(y_conv, 1), tf.argmax(self.labels, 1))
-            self.accuracy = tf.reduce_mean(
-                tf.cast(correct_prediction, tf.float32))
-
-
-def download_mnist_retry(data_dir, max_num_retries=20):
-    """Try to download mnist dataset and avoid errors"""
-    for _ in range(max_num_retries):
-        try:
-            return input_data.read_data_sets(data_dir, one_hot=True)
-        except tf.errors.AlreadyExistsError:
-            time.sleep(1)
-    raise Exception("Failed to download MNIST.")
-
-def main(params):
-    '''
-    Main function, build mnist network, run and send result to NNI.
-    '''
-    # Import data
-    mnist = download_mnist_retry(params['data_dir'])
-    print('Mnist download data done.')
-    logger.debug('Mnist download data done.')
-
-    # Create the model
-    # Build the graph for the deep net
-    mnist_network = MnistNetwork(channel_1_num=params['channel_1_num'],
-                                 channel_2_num=params['channel_2_num'],
-                                 conv_size=params['conv_size'],
-                                 hidden_size=params['hidden_size'],
-                                 pool_size=params['pool_size'],
-                                 learning_rate=params['learning_rate'])
-    mnist_network.build_network()
-    logger.debug('Mnist build network done.')
-
-    # Write log
-    graph_location = tempfile.mkdtemp()
-    logger.debug('Saving graph to: %s', graph_location)
-    train_writer = tf.summary.FileWriter(graph_location)
-    train_writer.add_graph(tf.get_default_graph())
-
-    test_acc = 0.0
-    with tf.Session() as sess:
-        sess.run(tf.global_variables_initializer())
-        for i in range(params['batch_num']):
-            """@nni.training_update(tf, sess)"""
-            batch = mnist.train.next_batch(params['batch_size'])
-            mnist_network.train_step.run(feed_dict={mnist_network.images: batch[0],
-                                                    mnist_network.labels: batch[1],
-                                                    mnist_network.keep_prob: 1 - params['dropout_rate']}
-                                        )
-
-            if i % 100 == 0:
-                test_acc = mnist_network.accuracy.eval(
-                    feed_dict={mnist_network.images: mnist.test.images,
-                               mnist_network.labels: mnist.test.labels,
-                               mnist_network.keep_prob: 1.0})
-
-                """@nni.report_intermediate_result(test_acc)"""
-                logger.debug('test accuracy %g', test_acc)
-                logger.debug('Pipe send intermediate result done.')
-
-        test_acc = mnist_network.accuracy.eval(
-            feed_dict={mnist_network.images: mnist.test.images,
-                       mnist_network.labels: mnist.test.labels,
-                       mnist_network.keep_prob: 1.0})
-
-        """@nni.report_final_result(test_acc)"""
-        logger.debug('Final result is %g', test_acc)
-        logger.debug('Send final result done.')
-
-def get_params():
-    ''' Get parameters from command line '''
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--data_dir", type=str, default='/tmp/tensorflow/mnist/input_data', help="data directory")
-    parser.add_argument("--dropout_rate", type=float, default=0.5, help="dropout rate")
-    parser.add_argument("--channel_1_num", type=int, default=32)
-    parser.add_argument("--channel_2_num", type=int, default=64)
-    parser.add_argument("--conv_size", type=int, default=5)
-    parser.add_argument("--pool_size", type=int, default=2)
-    parser.add_argument("--hidden_size", type=int, default=1024)
-    parser.add_argument("--learning_rate", type=float, default=1e-4)
-    parser.add_argument("--batch_num", type=int, default=2000)
-    parser.add_argument("--batch_size", type=int, default=32)
-
-    args, _ = parser.parse_known_args()
-    return args
-
-if __name__ == '__main__':
-    try:
-        params = vars(get_params())
-        main(params)
-    except Exception as exception:
-        logger.exception(exception)
-        raise

examples/trials/mnist-nas/enas_mode/operators.py

-import tensorflow as tf
-import math
-
-
-def weight_variable(shape):
-    """weight_variable generates a weight variable of a given shape."""
-    initial = tf.truncated_normal(shape, stddev=0.1)
-    return tf.Variable(initial)
-
-def bias_variable(shape):
-    """bias_variable generates a bias variable of a given shape."""
-    initial = tf.constant(0.1, shape=shape)
-    return tf.Variable(initial)
-
-def sum_op(inputs):
-    """sum_op"""
-    fixed_input = inputs[0][0]
-    optional_input = tf.reduce_sum(inputs[1], axis=0)
-    if len(optional_input.get_shape()) < 1:
-        return fixed_input
-    fixed_shape = fixed_input.get_shape().as_list()
-    optional_shape = optional_input.get_shape().as_list()
-    assert fixed_shape[1] == fixed_shape[2]
-    assert optional_shape[1] == optional_shape[2]
-    pool_size = math.ceil(optional_shape[1] / fixed_shape[1])
-    pool_out = tf.nn.avg_pool(optional_input, ksize=[1, pool_size, pool_size, 1], strides=[1, pool_size, pool_size, 1], padding='SAME')
-    conv_matrix = weight_variable([1, 1, optional_shape[3], fixed_shape[3]])
-    conv_out = tf.nn.conv2d(pool_out, conv_matrix, strides=[1, 1, 1, 1], padding='SAME')
-    return fixed_input + conv_out
-
-
-def conv2d(inputs, size=-1, in_ch=-1, out_ch=-1):
-    """conv2d returns a 2d convolution layer with full stride."""
-    x_input = sum_op(inputs)
-    if size in [1, 3]:
-        w_matrix = weight_variable([size, size, in_ch, out_ch])
-        return tf.nn.conv2d(x_input, w_matrix, strides=[1, 1, 1, 1], padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-def twice_conv2d(inputs, size=-1, in_ch=-1, out_ch=-1):
-    """twice_conv2d"""
-    x_input = sum_op(inputs)
-    if size in  [3, 7]:
-        w_matrix1 = weight_variable([1, size, in_ch, int(out_ch/2)])
-        out = tf.nn.conv2d(x_input, w_matrix1, strides=[1, 1, 1, 1], padding='SAME')
-        w_matrix2 = weight_variable([size, 1, int(out_ch/2), out_ch])
-        return tf.nn.conv2d(out, w_matrix2, strides=[1, 1, 1, 1], padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-def dilated_conv(inputs, size=3, in_ch=-1, out_ch=-1):
-    """dilated_conv"""
-    x_input = sum_op(inputs)
-    if size == 3:
-        w_matrix = weight_variable([size, size, in_ch, out_ch])
-        return tf.nn.atrous_conv2d(x_input, w_matrix, rate=2, padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-def separable_conv(inputs, size=-1, in_ch=-1, out_ch=-1):
-    """separable_conv"""
-    x_input = sum_op(inputs)
-    if size in [3, 5, 7]:
-        depth_matrix = weight_variable([size, size, in_ch, 1])
-        point_matrix = weight_variable([1, 1, 1*in_ch, out_ch])
-        return tf.nn.separable_conv2d(x_input, depth_matrix, point_matrix, strides=[1, 1, 1, 1], padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-
-def avg_pool(inputs, size=-1):
-    """avg_pool downsamples a feature map."""
-    x_input = sum_op(inputs)
-    if size in [3, 5, 7]:
-        return tf.nn.avg_pool(x_input, ksize=[1, size, size, 1], strides=[1, 1, 1, 1], padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-def max_pool(inputs, size=-1):
-    """max_pool downsamples a feature map."""
-    x_input = sum_op(inputs)
-    if size in [3, 5, 7]:
-        return tf.nn.max_pool(x_input, ksize=[1, size, size, 1], strides=[1, 1, 1, 1], padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-
-def post_process(inputs, ch_size=-1):
-    """post_process"""
-    x_input = inputs[0][0]
-    bias_matrix = bias_variable([ch_size])
-    return tf.nn.relu(x_input + bias_matrix)

examples/trials/mnist-nas/oneshot_mode/config_oneshot.yml

-authorName: default
-experimentName: example_mnist
-trialConcurrency: 1
-maxExecDuration: 1h
-maxTrialNum: 10
-#choice: local, remote, pai
-trainingServicePlatform: local
-#choice: true, false
-useAnnotation: true
-tuner:
-  codeDir: ../../../tuners/random_nas_tuner
-  classFileName: random_nas_tuner.py
-  className: RandomNASTuner
-trial:
-  command: python3 mnist-oneshot.py
-  codeDir: .
-  gpuNum: 0
-  nasMode: oneshot_mode

examples/trials/mnist-nas/oneshot_mode/mnist-oneshot.py

-"""A deep MNIST classifier using convolutional layers."""
-
-import argparse
-import logging
-import math
-import tempfile
-import time
-
-import tensorflow as tf
-from tensorflow.examples.tutorials.mnist import input_data
-
-import operators as op
-
-FLAGS = None
-
-logger = logging.getLogger('mnist_AutoML')
-
-
-class MnistNetwork(object):
-    '''
-    MnistNetwork is for initializing and building basic network for mnist.
-    '''
-    def __init__(self,
-                 channel_1_num,
-                 channel_2_num,
-                 conv_size,
-                 hidden_size,
-                 pool_size,
-                 learning_rate,
-                 x_dim=784,
-                 y_dim=10):
-        self.channel_1_num = channel_1_num
-        self.channel_2_num = channel_2_num
-        self.conv_size = conv_size
-        self.hidden_size = hidden_size
-        self.pool_size = pool_size
-        self.learning_rate = learning_rate
-        self.x_dim = x_dim
-        self.y_dim = y_dim
-
-        self.images = tf.placeholder(tf.float32, [None, self.x_dim], name='input_x')
-        self.labels = tf.placeholder(tf.float32, [None, self.y_dim], name='input_y')
-        self.keep_prob = tf.placeholder(tf.float32, name='keep_prob')
-
-        self.train_step = None
-        self.accuracy = None
-
-    def build_network(self):
-        '''
-        Building network for mnist, meanwhile specifying its neural architecture search space
-        '''
-
-        # Reshape to use within a convolutional neural net.
-        # Last dimension is for "features" - there is only one here, since images are
-        # grayscale -- it would be 3 for an RGB image, 4 for RGBA, etc.
-        with tf.name_scope('reshape'):
-            try:
-                input_dim = int(math.sqrt(self.x_dim))
-            except:
-                print(
-                    'input dim cannot be sqrt and reshape. input dim: ' + str(self.x_dim))
-                logger.debug(
-                    'input dim cannot be sqrt and reshape. input dim: %s', str(self.x_dim))
-                raise
-            x_image = tf.reshape(self.images, [-1, input_dim, input_dim, 1])
-
-        """@nni.mutable_layers(
-            {
-                layer_choice: [op.conv2d(size=1, in_ch=1, out_ch=self.channel_1_num),
-                               op.conv2d(size=3, in_ch=1, out_ch=self.channel_1_num),
-                               op.twice_conv2d(size=3, in_ch=1, out_ch=self.channel_1_num),
-                               op.twice_conv2d(size=7, in_ch=1, out_ch=self.channel_1_num),
-                               op.dilated_conv(in_ch=1, out_ch=self.channel_1_num),
-                               op.separable_conv(size=3, in_ch=1, out_ch=self.channel_1_num),
-                               op.separable_conv(size=5, in_ch=1, out_ch=self.channel_1_num),
-                               op.separable_conv(size=7, in_ch=1, out_ch=self.channel_1_num)],
-                fixed_inputs: [x_image],
-                layer_output: conv1_out
-            },
-            {
-                layer_choice: [op.post_process(ch_size=self.channel_1_num)],
-                fixed_inputs: [conv1_out],
-                layer_output: post1_out
-            },
-            {
-                layer_choice: [op.max_pool(size=3),
-                               op.max_pool(size=5),
-                               op.max_pool(size=7),
-                               op.avg_pool(size=3),
-                               op.avg_pool(size=5),
-                               op.avg_pool(size=7)],
-                fixed_inputs: [post1_out],
-                layer_output: pool1_out
-            },
-            {
-                layer_choice: [op.conv2d(size=1, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.conv2d(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.twice_conv2d(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.twice_conv2d(size=7, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.dilated_conv(in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.separable_conv(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.separable_conv(size=5, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
-                               op.separable_conv(size=7, in_ch=self.channel_1_num, out_ch=self.channel_2_num)],
-                fixed_inputs: [pool1_out],
-                optional_inputs: [post1_out],
-                optional_input_size: [0, 1],
-                layer_output: conv2_out
-            },
-            {
-                layer_choice: [op.post_process(ch_size=self.channel_2_num)],
-                fixed_inputs: [conv2_out],
-                layer_output: post2_out
-            },
-            {
-                layer_choice: [op.max_pool(size=3),
-                               op.max_pool(size=5),
-                               op.max_pool(size=7),
-                               op.avg_pool(size=3),
-                               op.avg_pool(size=5),
-                               op.avg_pool(size=7)],
-                fixed_inputs: [post2_out],
-                optional_inputs: [post1_out, pool1_out],
-                optional_input_size: [0, 1],
-                layer_output: pool2_out
-            }
-        )"""
-
-        # Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image
-        # is down to 7x7x64 feature maps -- maps this to 1024 features.
-        last_dim_list = pool2_out.get_shape().as_list()
-        assert(last_dim_list[1] == last_dim_list[2])
-        last_dim = last_dim_list[1]
-        with tf.name_scope('fc1'):
-            w_fc1 = op.weight_variable(
-                [last_dim * last_dim * self.channel_2_num, self.hidden_size])
-            b_fc1 = op.bias_variable([self.hidden_size])
-
-        h_pool2_flat = tf.reshape(
-            pool2_out, [-1, last_dim * last_dim * self.channel_2_num])
-        h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, w_fc1) + b_fc1)
-
-        # Dropout - controls the complexity of the model, prevents co-adaptation of features.
-        with tf.name_scope('dropout'):
-            h_fc1_drop = tf.nn.dropout(h_fc1, self.keep_prob)
-
-        # Map the 1024 features to 10 classes, one for each digit
-        with tf.name_scope('fc2'):
-            w_fc2 = op.weight_variable([self.hidden_size, self.y_dim])
-            b_fc2 = op.bias_variable([self.y_dim])
-            y_conv = tf.matmul(h_fc1_drop, w_fc2) + b_fc2
-
-        with tf.name_scope('loss'):
-            cross_entropy = tf.reduce_mean(
-                tf.nn.softmax_cross_entropy_with_logits(labels=self.labels, logits=y_conv))
-        with tf.name_scope('adam_optimizer'):
-            self.train_step = tf.train.AdamOptimizer(
-                self.learning_rate).minimize(cross_entropy)
-
-        with tf.name_scope('accuracy'):
-            correct_prediction = tf.equal(
-                tf.argmax(y_conv, 1), tf.argmax(self.labels, 1))
-            self.accuracy = tf.reduce_mean(
-                tf.cast(correct_prediction, tf.float32))
-
-
-def download_mnist_retry(data_dir, max_num_retries=20):
-    """Try to download mnist dataset and avoid errors"""
-    for _ in range(max_num_retries):
-        try:
-            return input_data.read_data_sets(data_dir, one_hot=True)
-        except tf.errors.AlreadyExistsError:
-            time.sleep(1)
-    raise Exception("Failed to download MNIST.")
-
-def main(params):
-    '''
-    Main function, build mnist network, run and send result to NNI.
-    '''
-    # Import data
-    mnist = download_mnist_retry(params['data_dir'])
-    print('Mnist download data done.')
-    logger.debug('Mnist download data done.')
-
-    # Create the model
-    # Build the graph for the deep net
-    mnist_network = MnistNetwork(channel_1_num=params['channel_1_num'],
-                                 channel_2_num=params['channel_2_num'],
-                                 conv_size=params['conv_size'],
-                                 hidden_size=params['hidden_size'],
-                                 pool_size=params['pool_size'],
-                                 learning_rate=params['learning_rate'])
-    mnist_network.build_network()
-    logger.debug('Mnist build network done.')
-
-    # Write log
-    graph_location = tempfile.mkdtemp()
-    logger.debug('Saving graph to: %s', graph_location)
-    train_writer = tf.summary.FileWriter(graph_location)
-    train_writer.add_graph(tf.get_default_graph())
-
-    test_acc = 0.0
-    with tf.Session() as sess:
-        sess.run(tf.global_variables_initializer())
-        for i in range(params['batch_num']):
-            batch = mnist.train.next_batch(params['batch_size'])
-            mnist_network.train_step.run(feed_dict={mnist_network.images: batch[0],
-                                                    mnist_network.labels: batch[1],
-                                                    mnist_network.keep_prob: 1 - params['dropout_rate']}
-                                        )
-
-            if i % 100 == 0:
-                test_acc = mnist_network.accuracy.eval(
-                    feed_dict={mnist_network.images: mnist.test.images,
-                               mnist_network.labels: mnist.test.labels,
-                               mnist_network.keep_prob: 1.0})
-
-                """@nni.report_intermediate_result(test_acc)"""
-                logger.debug('test accuracy %g', test_acc)
-                logger.debug('Pipe send intermediate result done.')
-
-        test_acc = mnist_network.accuracy.eval(
-            feed_dict={mnist_network.images: mnist.test.images,
-                       mnist_network.labels: mnist.test.labels,
-                       mnist_network.keep_prob: 1.0})
-
-        """@nni.report_final_result(test_acc)"""
-        logger.debug('Final result is %g', test_acc)
-        logger.debug('Send final result done.')
-
-def get_params():
-    ''' Get parameters from command line '''
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--data_dir", type=str, default='/tmp/tensorflow/mnist/input_data', help="data directory")
-    parser.add_argument("--dropout_rate", type=float, default=0.5, help="dropout rate")
-    parser.add_argument("--channel_1_num", type=int, default=32)
-    parser.add_argument("--channel_2_num", type=int, default=64)
-    parser.add_argument("--conv_size", type=int, default=5)
-    parser.add_argument("--pool_size", type=int, default=2)
-    parser.add_argument("--hidden_size", type=int, default=1024)
-    parser.add_argument("--learning_rate", type=float, default=1e-4)
-    parser.add_argument("--batch_num", type=int, default=2000)
-    parser.add_argument("--batch_size", type=int, default=32)
-
-    args, _ = parser.parse_known_args()
-    return args
-
-if __name__ == '__main__':
-    try:
-        params = vars(get_params())
-        main(params)
-    except Exception as exception:
-        logger.exception(exception)
-        raise

examples/trials/mnist-nas/oneshot_mode/operators.py

-import tensorflow as tf
-import math
-
-
-def weight_variable(shape):
-    """weight_variable generates a weight variable of a given shape."""
-    initial = tf.truncated_normal(shape, stddev=0.1)
-    return tf.Variable(initial)
-
-def bias_variable(shape):
-    """bias_variable generates a bias variable of a given shape."""
-    initial = tf.constant(0.1, shape=shape)
-    return tf.Variable(initial)
-
-def sum_op(inputs):
-    """sum_op"""
-    fixed_input = inputs[0][0]
-    optional_input = tf.concat(inputs[1], axis=3)
-    fixed_shape = fixed_input.get_shape().as_list()
-    optional_shape = optional_input.get_shape().as_list()
-    assert fixed_shape[1] == fixed_shape[2]
-    assert optional_shape[1] == optional_shape[2]
-    pool_size = math.ceil(optional_shape[1] / fixed_shape[1])
-    pool_out = tf.nn.avg_pool(optional_input, ksize=[1, pool_size, pool_size, 1], strides=[1, pool_size, pool_size, 1], padding='SAME')
-    conv_matrix = weight_variable([1, 1, optional_shape[3], fixed_shape[3]])
-    conv_out = tf.nn.conv2d(pool_out, conv_matrix, strides=[1, 1, 1, 1], padding='SAME')
-    return fixed_input + conv_out
-
-
-def conv2d(inputs, size=-1, in_ch=-1, out_ch=-1):
-    """conv2d returns a 2d convolution layer with full stride."""
-    if not inputs[1]:
-        x_input = inputs[0][0]
-    else:
-        x_input = sum_op(inputs)
-    if size in [1, 3]:
-        w_matrix = weight_variable([size, size, in_ch, out_ch])
-        return tf.nn.conv2d(x_input, w_matrix, strides=[1, 1, 1, 1], padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-def twice_conv2d(inputs, size=-1, in_ch=-1, out_ch=-1):
-    """twice_conv2d"""
-    if not inputs[1]:
-        x_input = inputs[0][0]
-    else:
-        x_input = sum_op(inputs)
-    if size in  [3, 7]:
-        w_matrix1 = weight_variable([1, size, in_ch, int(out_ch/2)])
-        out = tf.nn.conv2d(x_input, w_matrix1, strides=[1, 1, 1, 1], padding='SAME')
-        w_matrix2 = weight_variable([size, 1, int(out_ch/2), out_ch])
-        return tf.nn.conv2d(out, w_matrix2, strides=[1, 1, 1, 1], padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-def dilated_conv(inputs, size=3, in_ch=-1, out_ch=-1):
-    """dilated_conv"""
-    if not inputs[1]:
-        x_input = inputs[0][0]
-    else:
-        x_input = sum_op(inputs)
-    if size == 3:
-        w_matrix = weight_variable([size, size, in_ch, out_ch])
-        return tf.nn.atrous_conv2d(x_input, w_matrix, rate=2, padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-def separable_conv(inputs, size=-1, in_ch=-1, out_ch=-1):
-    """separable_conv"""
-    if not inputs[1]:
-        x_input = inputs[0][0]
-    else:
-        x_input = sum_op(inputs)
-    if size in [3, 5, 7]:
-        depth_matrix = weight_variable([size, size, in_ch, 1])
-        point_matrix = weight_variable([1, 1, 1*in_ch, out_ch])
-        return tf.nn.separable_conv2d(x_input, depth_matrix, point_matrix, strides=[1, 1, 1, 1], padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-
-def avg_pool(inputs, size=-1):
-    """avg_pool downsamples a feature map."""
-    if not inputs[1]:
-        x_input = inputs[0][0]
-    else:
-        x_input = sum_op(inputs)
-    if size in [3, 5, 7]:
-        return tf.nn.avg_pool(x_input, ksize=[1, size, size, 1], strides=[1, 1, 1, 1], padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-def max_pool(inputs, size=-1):
-    """max_pool downsamples a feature map."""
-    if not inputs[1]:
-        x_input = inputs[0][0]
-    else:
-        x_input = sum_op(inputs)
-    if size in [3, 5, 7]:
-        return tf.nn.max_pool(x_input, ksize=[1, size, size, 1], strides=[1, 1, 1, 1], padding='SAME')
-    else:
-        raise Exception("Unknown filter size: %d." % size)
-
-
-def post_process(inputs, ch_size=-1):
-    """post_process"""
-    x_input = inputs[0][0]
-    bias_matrix = bias_variable([ch_size])
-    return tf.nn.relu(x_input + bias_matrix)

examples/trials/nas_cifar10/README.md

-# Run Neural Architecture Search in NNI
- 
-Now we have an NAS example [NNI-NAS-Example](https://github.com/Crysple/NNI-NAS-Example) run in NNI using NAS interface from our contributors.	
-
-We have included its trial code in this folder, and provided example config files to show how to use PPO tuner to tune the trial code.
-
-To prepare for the dataset, please run `cd data && . download.sh`.
-
-Thanks our lovely contributors, and welcome more and more people to join us!
-

examples/trials/nas_cifar10/README_zh_CN.md

-# 在 NNI 中运行神经网络架构搜索
-
-参考 [NNI-NAS-Example](https://github.com/Crysple/NNI-NAS-Example)，来使用贡献者提供的 NAS 接口。
-
-此目录中包含了 Trial 代码，并提供了示例的配置文件来展示如何使用 PPO Tuner 来调优此 Trial 代码。
-
-运行下列代码来准备数据集 `cd data && . download.sh`.
-
-感谢可爱的志愿者，欢迎更多的人加入我们！
\ No newline at end of file

examples/trials/nas_cifar10/config_paiYarn_ppo.yml

-authorName: Unknown
-experimentName: enas_macro
-trialConcurrency: 20
-maxExecDuration: 2400h
-maxTrialNum: 20000
-#choice: local, remote
-trainingServicePlatform: paiYarn
-#choice: true, false
-useAnnotation: true
-multiPhase: false
-versionCheck: false
-nniManagerIp: 0.0.0.0
-tuner:
-  builtinTunerName: PPOTuner
-  classArgs:
-    optimize_mode: maximize
-    trials_per_update: 60
-    epochs_per_update: 20
-    minibatch_size: 6
-trial:
-  command: sh ./macro_cifar10_pai.sh
-  codeDir: ./
-  gpuNum: 1
-  cpuNum: 1
-  memoryMB: 8196
-  image: msranni/nni:latest
-  virtualCluster: nni
-paiYarnConfig:
-  userName: your_account
-  passWord: your_passwd
-  host: 0.0.0.0

examples/trials/nas_cifar10/config_pai_ppo.yml

-authorName: Unknown
-experimentName: enas_macro
-trialConcurrency: 20
-maxExecDuration: 2400h
-maxTrialNum: 20000
-#choice: local, remote
-trainingServicePlatform: pai
-#choice: true, false
-useAnnotation: true
-multiPhase: false
-versionCheck: false
-nniManagerIp: 0.0.0.0
-tuner:
-  builtinTunerName: PPOTuner
-  classArgs:
-    optimize_mode: maximize
-    trials_per_update: 60
-    epochs_per_update: 20
-    minibatch_size: 6
-trial:
-  command: sh ./macro_cifar10_pai.sh
-  codeDir: ./
-  gpuNum: 1
-  cpuNum: 1
-  memoryMB: 8196
-  image: msranni/nni:latest
-  virtualCluster: nni
-  nniManagerNFSMountPath: /home/user/mnt
-  containerNFSMountPath: /mnt/data/user
-  paiStoragePlugin: team_wise
-paiConfig:
-  userName: your_account
-  token: your_token
-  host: 0.0.0.0

examples/trials/nas_cifar10/config_ppo.yml

-authorName: Unknown
-experimentName: enas_macro
-trialConcurrency: 4
-maxExecDuration: 2400h
-maxTrialNum: 20000
-#choice: local, remote
-trainingServicePlatform: local
-#choice: true, false
-useAnnotation: true
-multiPhase: false
-tuner:
-  builtinTunerName: PPOTuner
-  classArgs:
-    optimize_mode: maximize
-    trials_per_update: 60
-    epochs_per_update: 12
-    minibatch_size: 10
-  #could use the No. 0 gpu for this tuner
-  #if want to specify multiple gpus, here is an example of specifying three gpus: 0,1,2
-  gpuIndices: 0
-trial:
-  command: sh ./macro_cifar10.sh
-  codeDir: ./
-  gpuNum: 1

examples/trials/nas_cifar10/data/download.sh

-wget https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz
-tar xzf cifar-10-python.tar.gz && mv cifar-10-batches-py cifar10
\ No newline at end of file

examples/trials/nas_cifar10/macro_cifar10.sh

-#!/bin/bash
-set -e
-export PYTHONPATH="$(pwd)"
-
-python3 src/cifar10/nni_child_cifar10.py \
-  --data_format="NCHW" \
-  --search_for="macro" \
-  --reset_output_dir \
-  --data_path="data/cifar10" \
-  --output_dir="outputs" \
-  --train_data_size=45000 \
-  --batch_size=100 \
-  --num_epochs=8 \
-  --log_every=50 \
-  --eval_every_epochs=1 \
-  --child_use_aux_heads \
-  --child_num_layers=12 \
-  --child_out_filters=36 \
-  --child_l2_reg=0.0002 \
-  --child_num_branches=6 \
-  --child_num_cell_layers=5 \
-  --child_keep_prob=0.50 \
-  --child_drop_path_keep_prob=0.60 \
-  --child_lr_cosine \
-  --child_lr_max=0.05 \
-  --child_lr_min=0.001 \
-  --child_lr_T_0=10 \
-  --child_lr_T_mul=2 \
-  --child_mode="subgraph" \
-  "$@"
-