Skip to content

GitLab

Unverified Commit 67287997 authored by SparkSnail's avatar SparkSnail Committed by GitHub
Browse files

Merge pull request #241 from microsoft/master 

merge master
parents b4773e1e f8d42a33
Hide whitespace changes
Inline Side-by-side
Showing with 8 additions and 1566 deletions
examples/model_compress/model_prune_torch.py
View file @ 67287997
...@@ -27,7 +27,7 @@ prune_config = { ...@@ -27,7 +27,7 @@ prune_config = {
'model_name': 'naive', 'model_name': 'naive',
'pruner_class': AGP_Pruner, 'pruner_class': AGP_Pruner,
'config_list': [{ 'config_list': [{
'initial_sparsity': 0, 'initial_sparsity': 0.,
'final_sparsity': 0.8, 'final_sparsity': 0.8,
'start_epoch': 0, 'start_epoch': 0,
'end_epoch': 10, 'end_epoch': 10,
......
examples/nas/darts/search.py
View file @ 67287997
...@@ -24,6 +24,7 @@ if __name__ == "__main__": ...@@ -24,6 +24,7 @@ if __name__ == "__main__":
parser.add_argument("--epochs", default=50, type=int) parser.add_argument("--epochs", default=50, type=int)
parser.add_argument("--channels", default=16, type=int) parser.add_argument("--channels", default=16, type=int)
parser.add_argument("--unrolled", default=False, action="store_true") parser.add_argument("--unrolled", default=False, action="store_true")
parser.add_argument("--visualization", default=False, action="store_true")
args = parser.parse_args() args = parser.parse_args()
dataset_train, dataset_valid = datasets.get_dataset("cifar10") dataset_train, dataset_valid = datasets.get_dataset("cifar10")
...@@ -45,4 +46,6 @@ if __name__ == "__main__": ...@@ -45,4 +46,6 @@ if __name__ == "__main__":
log_frequency=args.log_frequency, log_frequency=args.log_frequency,
unrolled=args.unrolled, unrolled=args.unrolled,
callbacks=[LRSchedulerCallback(lr_scheduler), ArchitectureCheckpoint("./checkpoints")]) callbacks=[LRSchedulerCallback(lr_scheduler), ArchitectureCheckpoint("./checkpoints")])
if args.visualization:
trainer.enable_visualization()
trainer.train() trainer.train()
examples/nas/enas/search.py
View file @ 67287997
...@@ -25,6 +25,7 @@ if __name__ == "__main__": ...@@ -25,6 +25,7 @@ if __name__ == "__main__":
parser.add_argument("--log-frequency", default=10, type=int) parser.add_argument("--log-frequency", default=10, type=int)
parser.add_argument("--search-for", choices=["macro", "micro"], default="macro") parser.add_argument("--search-for", choices=["macro", "micro"], default="macro")
parser.add_argument("--epochs", default=None, type=int, help="Number of epochs (default: macro 310, micro 150)") parser.add_argument("--epochs", default=None, type=int, help="Number of epochs (default: macro 310, micro 150)")
parser.add_argument("--visualization", default=False, action="store_true")
args = parser.parse_args() args = parser.parse_args()
dataset_train, dataset_valid = datasets.get_dataset("cifar10") dataset_train, dataset_valid = datasets.get_dataset("cifar10")
...@@ -55,4 +56,6 @@ if __name__ == "__main__": ...@@ -55,4 +56,6 @@ if __name__ == "__main__":
dataset_valid=dataset_valid, dataset_valid=dataset_valid,
log_frequency=args.log_frequency, log_frequency=args.log_frequency,
mutator=mutator) mutator=mutator)
if args.visualization:
trainer.enable_visualization()
trainer.train() trainer.train()
examples/nas/naive/train.py
View file @ 67287997
...@@ -68,5 +68,6 @@ if __name__ == "__main__": ...@@ -68,5 +68,6 @@ if __name__ == "__main__":
dataset_valid=dataset_valid, dataset_valid=dataset_valid,
batch_size=64, batch_size=64,
log_frequency=10) log_frequency=10)
trainer.enable_visualization()
trainer.train() trainer.train()
trainer.export("checkpoint.json") trainer.export("checkpoint.json")
examples/trials/mnist-nas/classic_mode/config_hpo.yml deleted 100644 → 0
View file @ b4773e1e
authorName: default
experimentName: example_mnist
trialConcurrency: 1
maxExecDuration: 1h
maxTrialNum: 10
#choice: local, remote, pai
trainingServicePlatform: local
#choice: true, false
useAnnotation: true
tuner:
builtinTunerName: TPE
trial:
command: python3 mnist.py --batch_num 200
codeDir: .
gpuNum: 0
nasMode: classic_mode
examples/trials/mnist-nas/classic_mode/config_nas.yml deleted 100644 → 0
View file @ b4773e1e
authorName: default
experimentName: example_mnist
trialConcurrency: 1
maxExecDuration: 1h
maxTrialNum: 10
#choice: local, remote, pai
trainingServicePlatform: local
#choice: true, false
useAnnotation: true
tuner:
codeDir: ../../../tuners/random_nas_tuner
classFileName: random_nas_tuner.py
className: RandomNASTuner
trial:
command: python3 mnist.py
codeDir: .
gpuNum: 0
nasMode: classic_mode
examples/trials/mnist-nas/classic_mode/mnist.py deleted 100644 → 0
View file @ b4773e1e
"""A deep MNIST classifier using convolutional layers."""
import argparse
import logging
import math
import tempfile
import time
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
import operators as op
FLAGS = None
logger = logging.getLogger('mnist_AutoML')
class MnistNetwork(object):
'''
MnistNetwork is for initializing and building basic network for mnist.
'''
def __init__(self,
channel_1_num,
channel_2_num,
conv_size,
hidden_size,
pool_size,
learning_rate,
x_dim=784,
y_dim=10):
self.channel_1_num = channel_1_num
self.channel_2_num = channel_2_num
self.conv_size = conv_size
self.hidden_size = hidden_size
self.pool_size = pool_size
self.learning_rate = learning_rate
self.x_dim = x_dim
self.y_dim = y_dim
self.images = tf.placeholder(tf.float32, [None, self.x_dim], name='input_x')
self.labels = tf.placeholder(tf.float32, [None, self.y_dim], name='input_y')
self.keep_prob = tf.placeholder(tf.float32, name='keep_prob')
self.train_step = None
self.accuracy = None
def build_network(self):
'''
Building network for mnist, meanwhile specifying its neural architecture search space
'''
# Reshape to use within a convolutional neural net.
# Last dimension is for "features" - there is only one here, since images are
# grayscale -- it would be 3 for an RGB image, 4 for RGBA, etc.
with tf.name_scope('reshape'):
try:
input_dim = int(math.sqrt(self.x_dim))
except:
print(
'input dim cannot be sqrt and reshape. input dim: ' + str(self.x_dim))
logger.debug(
'input dim cannot be sqrt and reshape. input dim: %s', str(self.x_dim))
raise
x_image = tf.reshape(self.images, [-1, input_dim, input_dim, 1])
"""@nni.mutable_layers(
{
layer_choice: [op.conv2d(size=1, in_ch=1, out_ch=self.channel_1_num),
op.conv2d(size=3, in_ch=1, out_ch=self.channel_1_num),
op.twice_conv2d(size=3, in_ch=1, out_ch=self.channel_1_num),
op.twice_conv2d(size=7, in_ch=1, out_ch=self.channel_1_num),
op.dilated_conv(in_ch=1, out_ch=self.channel_1_num),
op.separable_conv(size=3, in_ch=1, out_ch=self.channel_1_num),
op.separable_conv(size=5, in_ch=1, out_ch=self.channel_1_num),
op.separable_conv(size=7, in_ch=1, out_ch=self.channel_1_num)],
fixed_inputs: [x_image],
layer_output: conv1_out
},
{
layer_choice: [op.post_process(ch_size=self.channel_1_num)],
fixed_inputs: [conv1_out],
layer_output: post1_out
},
{
layer_choice: [op.max_pool(size=3),
op.max_pool(size=5),
op.max_pool(size=7),
op.avg_pool(size=3),
op.avg_pool(size=5),
op.avg_pool(size=7)],
fixed_inputs: [post1_out],
layer_output: pool1_out
},
{
layer_choice: [op.conv2d(size=1, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.conv2d(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.twice_conv2d(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.twice_conv2d(size=7, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.dilated_conv(in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.separable_conv(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.separable_conv(size=5, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.separable_conv(size=7, in_ch=self.channel_1_num, out_ch=self.channel_2_num)],
fixed_inputs: [pool1_out],
optional_inputs: [post1_out],
optional_input_size: [0, 1],
layer_output: conv2_out
},
{
layer_choice: [op.post_process(ch_size=self.channel_2_num)],
fixed_inputs: [conv2_out],
layer_output: post2_out
},
{
layer_choice: [op.max_pool(size=3),
op.max_pool(size=5),
op.max_pool(size=7),
op.avg_pool(size=3),
op.avg_pool(size=5),
op.avg_pool(size=7)],
fixed_inputs: [post2_out],
optional_inputs: [post1_out, pool1_out],
optional_input_size: [0, 1],
layer_output: pool2_out
}
)"""
# Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image
# is down to 7x7x64 feature maps -- maps this to 1024 features.
last_dim_list = pool2_out.get_shape().as_list()
assert(last_dim_list[1] == last_dim_list[2])
last_dim = last_dim_list[1]
with tf.name_scope('fc1'):
w_fc1 = op.weight_variable(
[last_dim * last_dim * self.channel_2_num, self.hidden_size])
b_fc1 = op.bias_variable([self.hidden_size])
h_pool2_flat = tf.reshape(
pool2_out, [-1, last_dim * last_dim * self.channel_2_num])
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, w_fc1) + b_fc1)
# Dropout - controls the complexity of the model, prevents co-adaptation of features.
with tf.name_scope('dropout'):
h_fc1_drop = tf.nn.dropout(h_fc1, self.keep_prob)
# Map the 1024 features to 10 classes, one for each digit
with tf.name_scope('fc2'):
w_fc2 = op.weight_variable([self.hidden_size, self.y_dim])
b_fc2 = op.bias_variable([self.y_dim])
y_conv = tf.matmul(h_fc1_drop, w_fc2) + b_fc2
with tf.name_scope('loss'):
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=self.labels, logits=y_conv))
with tf.name_scope('adam_optimizer'):
self.train_step = tf.train.AdamOptimizer(
self.learning_rate).minimize(cross_entropy)
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(
tf.argmax(y_conv, 1), tf.argmax(self.labels, 1))
self.accuracy = tf.reduce_mean(
tf.cast(correct_prediction, tf.float32))
def download_mnist_retry(data_dir, max_num_retries=20):
"""Try to download mnist dataset and avoid errors"""
for _ in range(max_num_retries):
try:
return input_data.read_data_sets(data_dir, one_hot=True)
except tf.errors.AlreadyExistsError:
time.sleep(1)
raise Exception("Failed to download MNIST.")
def main(params):
'''
Main function, build mnist network, run and send result to NNI.
'''
# Import data
mnist = download_mnist_retry(params['data_dir'])
print('Mnist download data done.')
logger.debug('Mnist download data done.')
# Create the model
# Build the graph for the deep net
mnist_network = MnistNetwork(channel_1_num=params['channel_1_num'],
channel_2_num=params['channel_2_num'],
conv_size=params['conv_size'],
hidden_size=params['hidden_size'],
pool_size=params['pool_size'],
learning_rate=params['learning_rate'])
mnist_network.build_network()
logger.debug('Mnist build network done.')
# Write log
graph_location = tempfile.mkdtemp()
logger.debug('Saving graph to: %s', graph_location)
train_writer = tf.summary.FileWriter(graph_location)
train_writer.add_graph(tf.get_default_graph())
test_acc = 0.0
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in range(params['batch_num']):
batch = mnist.train.next_batch(params['batch_size'])
mnist_network.train_step.run(feed_dict={mnist_network.images: batch[0],
mnist_network.labels: batch[1],
mnist_network.keep_prob: 1 - params['dropout_rate']}
)
if i % 100 == 0:
test_acc = mnist_network.accuracy.eval(
feed_dict={mnist_network.images: mnist.test.images,
mnist_network.labels: mnist.test.labels,
mnist_network.keep_prob: 1.0})
"""@nni.report_intermediate_result(test_acc)"""
logger.debug('test accuracy %g', test_acc)
logger.debug('Pipe send intermediate result done.')
test_acc = mnist_network.accuracy.eval(
feed_dict={mnist_network.images: mnist.test.images,
mnist_network.labels: mnist.test.labels,
mnist_network.keep_prob: 1.0})
"""@nni.report_final_result(test_acc)"""
logger.debug('Final result is %g', test_acc)
logger.debug('Send final result done.')
def get_params():
''' Get parameters from command line '''
parser = argparse.ArgumentParser()
parser.add_argument("--data_dir", type=str, default='/tmp/tensorflow/mnist/input_data', help="data directory")
parser.add_argument("--dropout_rate", type=float, default=0.5, help="dropout rate")
parser.add_argument("--channel_1_num", type=int, default=32)
parser.add_argument("--channel_2_num", type=int, default=64)
parser.add_argument("--conv_size", type=int, default=5)
parser.add_argument("--pool_size", type=int, default=2)
parser.add_argument("--hidden_size", type=int, default=1024)
parser.add_argument("--learning_rate", type=float, default=1e-4)
parser.add_argument("--batch_num", type=int, default=2000)
parser.add_argument("--batch_size", type=int, default=32)
args, _ = parser.parse_known_args()
return args
if __name__ == '__main__':
try:
params = vars(get_params())
main(params)
except Exception as exception:
logger.exception(exception)
raise
examples/trials/mnist-nas/classic_mode/operators.py deleted 100644 → 0
View file @ b4773e1e
import tensorflow as tf
import math
def weight_variable(shape):
"""weight_variable generates a weight variable of a given shape."""
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial)
def bias_variable(shape):
"""bias_variable generates a bias variable of a given shape."""
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial)
def sum_op(inputs):
"""sum_op"""
fixed_input = inputs[0][0]
optional_input = inputs[1][0]
fixed_shape = fixed_input.get_shape().as_list()
optional_shape = optional_input.get_shape().as_list()
assert fixed_shape[1] == fixed_shape[2]
assert optional_shape[1] == optional_shape[2]
pool_size = math.ceil(optional_shape[1] / fixed_shape[1])
pool_out = tf.nn.avg_pool(optional_input, ksize=[1, pool_size, pool_size, 1], strides=[1, pool_size, pool_size, 1], padding='SAME')
conv_matrix = weight_variable([1, 1, optional_shape[3], fixed_shape[3]])
conv_out = tf.nn.conv2d(pool_out, conv_matrix, strides=[1, 1, 1, 1], padding='SAME')
return fixed_input + conv_out
def conv2d(inputs, size=-1, in_ch=-1, out_ch=-1):
"""conv2d returns a 2d convolution layer with full stride."""
if not inputs[1]:
x_input = inputs[0][0]
else:
x_input = sum_op(inputs)
if size in [1, 3]:
w_matrix = weight_variable([size, size, in_ch, out_ch])
return tf.nn.conv2d(x_input, w_matrix, strides=[1, 1, 1, 1], padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def twice_conv2d(inputs, size=-1, in_ch=-1, out_ch=-1):
"""twice_conv2d"""
if not inputs[1]:
x_input = inputs[0][0]
else:
x_input = sum_op(inputs)
if size in [3, 7]:
w_matrix1 = weight_variable([1, size, in_ch, int(out_ch/2)])
out = tf.nn.conv2d(x_input, w_matrix1, strides=[1, 1, 1, 1], padding='SAME')
w_matrix2 = weight_variable([size, 1, int(out_ch/2), out_ch])
return tf.nn.conv2d(out, w_matrix2, strides=[1, 1, 1, 1], padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def dilated_conv(inputs, size=3, in_ch=-1, out_ch=-1):
"""dilated_conv"""
if not inputs[1]:
x_input = inputs[0][0]
else:
x_input = sum_op(inputs)
if size == 3:
w_matrix = weight_variable([size, size, in_ch, out_ch])
return tf.nn.atrous_conv2d(x_input, w_matrix, rate=2, padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def separable_conv(inputs, size=-1, in_ch=-1, out_ch=-1):
"""separable_conv"""
if not inputs[1]:
x_input = inputs[0][0]
else:
x_input = sum_op(inputs)
if size in [3, 5, 7]:
depth_matrix = weight_variable([size, size, in_ch, 1])
point_matrix = weight_variable([1, 1, 1*in_ch, out_ch])
return tf.nn.separable_conv2d(x_input, depth_matrix, point_matrix, strides=[1, 1, 1, 1], padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def avg_pool(inputs, size=-1):
"""avg_pool downsamples a feature map."""
if not inputs[1]:
x_input = inputs[0][0]
else:
x_input = sum_op(inputs)
if size in [3, 5, 7]:
return tf.nn.avg_pool(x_input, ksize=[1, size, size, 1], strides=[1, size, size, 1], padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def max_pool(inputs, size=-1):
"""max_pool downsamples a feature map."""
if not inputs[1]:
x_input = inputs[0][0]
else:
x_input = sum_op(inputs)
if size in [3, 5, 7]:
return tf.nn.max_pool(x_input, ksize=[1, size, size, 1], strides=[1, size, size, 1], padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def post_process(inputs, ch_size=-1):
"""post_process"""
x_input = inputs[0][0]
bias_matrix = bias_variable([ch_size])
return tf.nn.relu(x_input + bias_matrix)
examples/trials/mnist-nas/config_ppo.yml deleted 100644 → 0
View file @ b4773e1e
authorName: NNI-example
experimentName: example_mnist
trialConcurrency: 1
maxExecDuration: 100h
maxTrialNum: 10000
#choice: local, remote, pai
trainingServicePlatform: local
#choice: true, false
useAnnotation: true
tuner:
#choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner
#SMAC, PPO (SMAC and PPO should be installed through nnictl)
builtinTunerName: PPOTuner
classArgs:
optimize_mode: maximize
trial:
command: python3 mnist.py
codeDir: .
gpuNum: 0
examples/trials/mnist-nas/darts_mode/config_darts.yml deleted 100644 → 0
View file @ b4773e1e
authorName: default
experimentName: example_mnist
trialConcurrency: 1
maxExecDuration: 1h
maxTrialNum: 10
#choice: local, remote, pai
trainingServicePlatform: local
#choice: true, false
useAnnotation: true
tuner:
codeDir: ../../../tuners/random_nas_tuner
classFileName: random_nas_tuner.py
className: RandomNASTuner
trial:
command: python3 mnist-darts.py
codeDir: .
gpuNum: 0
nasMode: oneshot_mode
examples/trials/mnist-nas/darts_mode/mnist-darts.py deleted 100644 → 0
View file @ b4773e1e
"""A deep MNIST classifier using convolutional layers."""
import argparse
import logging
import math
import tempfile
import time
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
import operators as op
FLAGS = None
logger = logging.getLogger('mnist_AutoML')
class MnistNetwork(object):
'''
MnistNetwork is for initializing and building basic network for mnist.
'''
def __init__(self,
channel_1_num,
channel_2_num,
conv_size,
hidden_size,
pool_size,
learning_rate,
x_dim=784,
y_dim=10):
self.channel_1_num = channel_1_num
self.channel_2_num = channel_2_num
self.conv_size = conv_size
self.hidden_size = hidden_size
self.pool_size = pool_size
self.learning_rate = learning_rate
self.x_dim = x_dim
self.y_dim = y_dim
self.images = tf.placeholder(tf.float32, [None, self.x_dim], name='input_x')
self.labels = tf.placeholder(tf.float32, [None, self.y_dim], name='input_y')
self.keep_prob = tf.placeholder(tf.float32, name='keep_prob')
self.train_step = None
self.accuracy = None
def build_network(self):
'''
Building network for mnist, meanwhile specifying its neural architecture search space
'''
# Reshape to use within a convolutional neural net.
# Last dimension is for "features" - there is only one here, since images are
# grayscale -- it would be 3 for an RGB image, 4 for RGBA, etc.
with tf.name_scope('reshape'):
try:
input_dim = int(math.sqrt(self.x_dim))
except:
print(
'input dim cannot be sqrt and reshape. input dim: ' + str(self.x_dim))
logger.debug(
'input dim cannot be sqrt and reshape. input dim: %s', str(self.x_dim))
raise
x_image = tf.reshape(self.images, [-1, input_dim, input_dim, 1])
"""@nni.mutable_layers(
{
layer_choice: [op.conv2d(size=1, in_ch=1, out_ch=self.channel_1_num),
op.conv2d(size=3, in_ch=1, out_ch=self.channel_1_num),
op.twice_conv2d(size=3, in_ch=1, out_ch=self.channel_1_num),
op.twice_conv2d(size=7, in_ch=1, out_ch=self.channel_1_num),
op.dilated_conv(in_ch=1, out_ch=self.channel_1_num),
op.separable_conv(size=3, in_ch=1, out_ch=self.channel_1_num),
op.separable_conv(size=5, in_ch=1, out_ch=self.channel_1_num),
op.separable_conv(size=7, in_ch=1, out_ch=self.channel_1_num)],
fixed_inputs: [x_image],
layer_output: conv1_out
},
{
layer_choice: [op.post_process(ch_size=self.channel_1_num)],
fixed_inputs: [conv1_out],
layer_output: post1_out
},
{
layer_choice: [op.max_pool(size=3),
op.max_pool(size=5),
op.max_pool(size=7),
op.avg_pool(size=3),
op.avg_pool(size=5),
op.avg_pool(size=7)],
fixed_inputs: [post1_out],
layer_output: pool1_out
},
{
layer_choice: [op.conv2d(size=1, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.conv2d(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.twice_conv2d(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.twice_conv2d(size=7, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.dilated_conv(in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.separable_conv(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.separable_conv(size=5, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.separable_conv(size=7, in_ch=self.channel_1_num, out_ch=self.channel_2_num)],
fixed_inputs: [pool1_out],
optional_inputs: [post1_out],
optional_input_size: [0, 1],
layer_output: conv2_out
},
{
layer_choice: [op.post_process(ch_size=self.channel_2_num)],
fixed_inputs: [conv2_out],
layer_output: post2_out
},
{
layer_choice: [op.max_pool(size=3),
op.max_pool(size=5),
op.max_pool(size=7),
op.avg_pool(size=3),
op.avg_pool(size=5),
op.avg_pool(size=7)],
fixed_inputs: [post2_out],
optional_inputs: [post1_out, pool1_out],
optional_input_size: [0, 1],
layer_output: pool2_out
}
)"""
# Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image
# is down to 7x7x64 feature maps -- maps this to 1024 features.
last_dim_list = pool2_out.get_shape().as_list()
assert(last_dim_list[1] == last_dim_list[2])
last_dim = last_dim_list[1]
with tf.name_scope('fc1'):
w_fc1 = op.weight_variable(
[last_dim * last_dim * self.channel_2_num, self.hidden_size])
b_fc1 = op.bias_variable([self.hidden_size])
h_pool2_flat = tf.reshape(
pool2_out, [-1, last_dim * last_dim * self.channel_2_num])
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, w_fc1) + b_fc1)
# Dropout - controls the complexity of the model, prevents co-adaptation of features.
with tf.name_scope('dropout'):
h_fc1_drop = tf.nn.dropout(h_fc1, self.keep_prob)
# Map the 1024 features to 10 classes, one for each digit
with tf.name_scope('fc2'):
w_fc2 = op.weight_variable([self.hidden_size, self.y_dim])
b_fc2 = op.bias_variable([self.y_dim])
y_conv = tf.matmul(h_fc1_drop, w_fc2) + b_fc2
with tf.name_scope('loss'):
self.cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=self.labels, logits=y_conv))
with tf.name_scope('adam_optimizer'):
self.train_step = tf.train.AdamOptimizer(
self.learning_rate).minimize(self.cross_entropy)
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(
tf.argmax(y_conv, 1), tf.argmax(self.labels, 1))
self.accuracy = tf.reduce_mean(
tf.cast(correct_prediction, tf.float32))
def download_mnist_retry(data_dir, max_num_retries=20):
"""Try to download mnist dataset and avoid errors"""
for _ in range(max_num_retries):
try:
return input_data.read_data_sets(data_dir, one_hot=True)
except tf.errors.AlreadyExistsError:
time.sleep(1)
raise Exception("Failed to download MNIST.")
def main(params):
'''
Main function, build mnist network, run and send result to NNI.
'''
# Import data
mnist = download_mnist_retry(params['data_dir'])
print('Mnist download data done.')
logger.debug('Mnist download data done.')
# Create the model
# Build the graph for the deep net
mnist_network = MnistNetwork(channel_1_num=params['channel_1_num'],
channel_2_num=params['channel_2_num'],
conv_size=params['conv_size'],
hidden_size=params['hidden_size'],
pool_size=params['pool_size'],
learning_rate=params['learning_rate'])
mnist_network.build_network()
logger.debug('Mnist build network done.')
# Write log
graph_location = tempfile.mkdtemp()
logger.debug('Saving graph to: %s', graph_location)
train_writer = tf.summary.FileWriter(graph_location)
train_writer.add_graph(tf.get_default_graph())
test_acc = 0.0
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in range(params['batch_num']):
batch = mnist.train.next_batch(params['batch_size'])
feed_dict={mnist_network.images: batch[0],
mnist_network.labels: batch[1],
mnist_network.keep_prob: 1 - params['dropout_rate']}
"""@nni.training_update(tf, sess, mnist_network.cross_entropy)"""
batch = mnist.train.next_batch(params['batch_size'])
feed_dict={mnist_network.images: batch[0],
mnist_network.labels: batch[1],
mnist_network.keep_prob: 1 - params['dropout_rate']}
mnist_network.train_step.run(feed_dict=feed_dict)
if i % 100 == 0:
test_acc = mnist_network.accuracy.eval(
feed_dict={mnist_network.images: mnist.test.images,
mnist_network.labels: mnist.test.labels,
mnist_network.keep_prob: 1.0})
"""@nni.report_intermediate_result(test_acc)"""
logger.debug('test accuracy %g', test_acc)
logger.debug('Pipe send intermediate result done.')
test_acc = mnist_network.accuracy.eval(
feed_dict={mnist_network.images: mnist.test.images,
mnist_network.labels: mnist.test.labels,
mnist_network.keep_prob: 1.0})
"""@nni.report_final_result(test_acc)"""
logger.debug('Final result is %g', test_acc)
logger.debug('Send final result done.')
def get_params():
''' Get parameters from command line '''
parser = argparse.ArgumentParser()
parser.add_argument("--data_dir", type=str, default='/tmp/tensorflow/mnist/input_data', help="data directory")
parser.add_argument("--dropout_rate", type=float, default=0.5, help="dropout rate")
parser.add_argument("--channel_1_num", type=int, default=32)
parser.add_argument("--channel_2_num", type=int, default=64)
parser.add_argument("--conv_size", type=int, default=5)
parser.add_argument("--pool_size", type=int, default=2)
parser.add_argument("--hidden_size", type=int, default=1024)
parser.add_argument("--learning_rate", type=float, default=1e-4)
parser.add_argument("--batch_num", type=int, default=2000)
parser.add_argument("--batch_size", type=int, default=32)
args, _ = parser.parse_known_args()
return args
if __name__ == '__main__':
try:
params = vars(get_params())
main(params)
except Exception as exception:
logger.exception(exception)
raise
examples/trials/mnist-nas/darts_mode/operators.py deleted 100644 → 0
View file @ b4773e1e
import tensorflow as tf
import math
def weight_variable(shape):
"""weight_variable generates a weight variable of a given shape."""
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial)
def bias_variable(shape):
"""bias_variable generates a bias variable of a given shape."""
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial)
def sum_op(inputs):
"""sum_op"""
fixed_input = inputs[0][0]
optional_input = tf.concat(inputs[1], axis=3)
fixed_shape = fixed_input.get_shape().as_list()
optional_shape = optional_input.get_shape().as_list()
assert fixed_shape[1] == fixed_shape[2]
assert optional_shape[1] == optional_shape[2]
pool_size = math.ceil(optional_shape[1] / fixed_shape[1])
pool_out = tf.nn.avg_pool(optional_input, ksize=[1, pool_size, pool_size, 1], strides=[1, pool_size, pool_size, 1], padding='SAME')
conv_matrix = weight_variable([1, 1, optional_shape[3], fixed_shape[3]])
conv_out = tf.nn.conv2d(pool_out, conv_matrix, strides=[1, 1, 1, 1], padding='SAME')
return fixed_input + conv_out
def conv2d(inputs, size=-1, in_ch=-1, out_ch=-1):
"""conv2d returns a 2d convolution layer with full stride."""
if not inputs[1]:
x_input = inputs[0][0]
else:
x_input = sum_op(inputs)
if size in [1, 3]:
w_matrix = weight_variable([size, size, in_ch, out_ch])
return tf.nn.conv2d(x_input, w_matrix, strides=[1, 1, 1, 1], padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def twice_conv2d(inputs, size=-1, in_ch=-1, out_ch=-1):
"""twice_conv2d"""
if not inputs[1]:
x_input = inputs[0][0]
else:
x_input = sum_op(inputs)
if size in [3, 7]:
w_matrix1 = weight_variable([1, size, in_ch, int(out_ch/2)])
out = tf.nn.conv2d(x_input, w_matrix1, strides=[1, 1, 1, 1], padding='SAME')
w_matrix2 = weight_variable([size, 1, int(out_ch/2), out_ch])
return tf.nn.conv2d(out, w_matrix2, strides=[1, 1, 1, 1], padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def dilated_conv(inputs, size=3, in_ch=-1, out_ch=-1):
"""dilated_conv"""
if not inputs[1]:
x_input = inputs[0][0]
else:
x_input = sum_op(inputs)
if size == 3:
w_matrix = weight_variable([size, size, in_ch, out_ch])
return tf.nn.atrous_conv2d(x_input, w_matrix, rate=2, padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def separable_conv(inputs, size=-1, in_ch=-1, out_ch=-1):
"""separable_conv"""
if not inputs[1]:
x_input = inputs[0][0]
else:
x_input = sum_op(inputs)
if size in [3, 5, 7]:
depth_matrix = weight_variable([size, size, in_ch, 1])
point_matrix = weight_variable([1, 1, 1*in_ch, out_ch])
return tf.nn.separable_conv2d(x_input, depth_matrix, point_matrix, strides=[1, 1, 1, 1], padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def avg_pool(inputs, size=-1):
"""avg_pool downsamples a feature map."""
if not inputs[1]:
x_input = inputs[0][0]
else:
x_input = sum_op(inputs)
if size in [3, 5, 7]:
return tf.nn.avg_pool(x_input, ksize=[1, size, size, 1], strides=[1, 1, 1, 1], padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def max_pool(inputs, size=-1):
"""max_pool downsamples a feature map."""
if not inputs[1]:
x_input = inputs[0][0]
else:
x_input = sum_op(inputs)
if size in [3, 5, 7]:
return tf.nn.max_pool(x_input, ksize=[1, size, size, 1], strides=[1, 1, 1, 1], padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def post_process(inputs, ch_size=-1):
"""post_process"""
x_input = inputs[0][0]
bias_matrix = bias_variable([ch_size])
return tf.nn.relu(x_input + bias_matrix)
examples/trials/mnist-nas/enas_mode/config_enas.yml deleted 100644 → 0
View file @ b4773e1e
authorName: default
experimentName: example_mnist
trialConcurrency: 1
maxExecDuration: 1h
maxTrialNum: 10
#choice: local, remote, pai
trainingServicePlatform: local
#choice: true, false
useAnnotation: true
multiPhase: true
tuner:
codeDir: ../../../tuners/random_nas_tuner
classFileName: random_nas_tuner.py
className: RandomNASTuner
trial:
command: python3 mnist-enas.py
codeDir: .
gpuNum: 0
nasMode: enas_mode
examples/trials/mnist-nas/enas_mode/mnist-enas.py deleted 100644 → 0
View file @ b4773e1e
"""A deep MNIST classifier using convolutional layers."""
import argparse
import logging
import math
import tempfile
import time
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
import operators as op
FLAGS = None
logger = logging.getLogger('mnist_AutoML')
class MnistNetwork(object):
'''
MnistNetwork is for initializing and building basic network for mnist.
'''
def __init__(self,
channel_1_num,
channel_2_num,
conv_size,
hidden_size,
pool_size,
learning_rate,
x_dim=784,
y_dim=10):
self.channel_1_num = channel_1_num
self.channel_2_num = channel_2_num
self.conv_size = conv_size
self.hidden_size = hidden_size
self.pool_size = pool_size
self.learning_rate = learning_rate
self.x_dim = x_dim
self.y_dim = y_dim
self.images = tf.placeholder(tf.float32, [None, self.x_dim], name='input_x')
self.labels = tf.placeholder(tf.float32, [None, self.y_dim], name='input_y')
self.keep_prob = tf.placeholder(tf.float32, name='keep_prob')
self.train_step = None
self.accuracy = None
def build_network(self):
'''
Building network for mnist, meanwhile specifying its neural architecture search space
'''
# Reshape to use within a convolutional neural net.
# Last dimension is for "features" - there is only one here, since images are
# grayscale -- it would be 3 for an RGB image, 4 for RGBA, etc.
with tf.name_scope('reshape'):
try:
input_dim = int(math.sqrt(self.x_dim))
except:
print(
'input dim cannot be sqrt and reshape. input dim: ' + str(self.x_dim))
logger.debug(
'input dim cannot be sqrt and reshape. input dim: %s', str(self.x_dim))
raise
x_image = tf.reshape(self.images, [-1, input_dim, input_dim, 1])
"""@nni.mutable_layers(
{
layer_choice: [op.conv2d(size=1, in_ch=1, out_ch=self.channel_1_num),
op.conv2d(size=3, in_ch=1, out_ch=self.channel_1_num),
op.twice_conv2d(size=3, in_ch=1, out_ch=self.channel_1_num),
op.twice_conv2d(size=7, in_ch=1, out_ch=self.channel_1_num),
op.dilated_conv(in_ch=1, out_ch=self.channel_1_num),
op.separable_conv(size=3, in_ch=1, out_ch=self.channel_1_num),
op.separable_conv(size=5, in_ch=1, out_ch=self.channel_1_num),
op.separable_conv(size=7, in_ch=1, out_ch=self.channel_1_num)],
fixed_inputs: [x_image],
layer_output: conv1_out
},
{
layer_choice: [op.post_process(ch_size=self.channel_1_num)],
fixed_inputs: [conv1_out],
layer_output: post1_out
},
{
layer_choice: [op.max_pool(size=3),
op.max_pool(size=5),
op.max_pool(size=7),
op.avg_pool(size=3),
op.avg_pool(size=5),
op.avg_pool(size=7)],
fixed_inputs: [post1_out],
layer_output: pool1_out
},
{
layer_choice: [op.conv2d(size=1, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.conv2d(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.twice_conv2d(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.twice_conv2d(size=7, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.dilated_conv(in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.separable_conv(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.separable_conv(size=5, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.separable_conv(size=7, in_ch=self.channel_1_num, out_ch=self.channel_2_num)],
fixed_inputs: [pool1_out],
optional_inputs: [post1_out],
optional_input_size: [0, 1],
layer_output: conv2_out
},
{
layer_choice: [op.post_process(ch_size=self.channel_2_num)],
fixed_inputs: [conv2_out],
layer_output: post2_out
},
{
layer_choice: [op.max_pool(size=3),
op.max_pool(size=5),
op.max_pool(size=7),
op.avg_pool(size=3),
op.avg_pool(size=5),
op.avg_pool(size=7)],
fixed_inputs: [post2_out],
optional_inputs: [post1_out, pool1_out],
optional_input_size: [0, 1],
layer_output: pool2_out
}
)"""
# Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image
# is down to 7x7x64 feature maps -- maps this to 1024 features.
last_dim_list = pool2_out.get_shape().as_list()
assert(last_dim_list[1] == last_dim_list[2])
last_dim = last_dim_list[1]
with tf.name_scope('fc1'):
w_fc1 = op.weight_variable(
[last_dim * last_dim * self.channel_2_num, self.hidden_size])
b_fc1 = op.bias_variable([self.hidden_size])
h_pool2_flat = tf.reshape(
pool2_out, [-1, last_dim * last_dim * self.channel_2_num])
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, w_fc1) + b_fc1)
# Dropout - controls the complexity of the model, prevents co-adaptation of features.
with tf.name_scope('dropout'):
h_fc1_drop = tf.nn.dropout(h_fc1, self.keep_prob)
# Map the 1024 features to 10 classes, one for each digit
with tf.name_scope('fc2'):
w_fc2 = op.weight_variable([self.hidden_size, self.y_dim])
b_fc2 = op.bias_variable([self.y_dim])
y_conv = tf.matmul(h_fc1_drop, w_fc2) + b_fc2
with tf.name_scope('loss'):
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=self.labels, logits=y_conv))
with tf.name_scope('adam_optimizer'):
self.train_step = tf.train.AdamOptimizer(
self.learning_rate).minimize(cross_entropy)
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(
tf.argmax(y_conv, 1), tf.argmax(self.labels, 1))
self.accuracy = tf.reduce_mean(
tf.cast(correct_prediction, tf.float32))
def download_mnist_retry(data_dir, max_num_retries=20):
"""Try to download mnist dataset and avoid errors"""
for _ in range(max_num_retries):
try:
return input_data.read_data_sets(data_dir, one_hot=True)
except tf.errors.AlreadyExistsError:
time.sleep(1)
raise Exception("Failed to download MNIST.")
def main(params):
'''
Main function, build mnist network, run and send result to NNI.
'''
# Import data
mnist = download_mnist_retry(params['data_dir'])
print('Mnist download data done.')
logger.debug('Mnist download data done.')
# Create the model
# Build the graph for the deep net
mnist_network = MnistNetwork(channel_1_num=params['channel_1_num'],
channel_2_num=params['channel_2_num'],
conv_size=params['conv_size'],
hidden_size=params['hidden_size'],
pool_size=params['pool_size'],
learning_rate=params['learning_rate'])
mnist_network.build_network()
logger.debug('Mnist build network done.')
# Write log
graph_location = tempfile.mkdtemp()
logger.debug('Saving graph to: %s', graph_location)
train_writer = tf.summary.FileWriter(graph_location)
train_writer.add_graph(tf.get_default_graph())
test_acc = 0.0
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in range(params['batch_num']):
"""@nni.training_update(tf, sess)"""
batch = mnist.train.next_batch(params['batch_size'])
mnist_network.train_step.run(feed_dict={mnist_network.images: batch[0],
mnist_network.labels: batch[1],
mnist_network.keep_prob: 1 - params['dropout_rate']}
)
if i % 100 == 0:
test_acc = mnist_network.accuracy.eval(
feed_dict={mnist_network.images: mnist.test.images,
mnist_network.labels: mnist.test.labels,
mnist_network.keep_prob: 1.0})
"""@nni.report_intermediate_result(test_acc)"""
logger.debug('test accuracy %g', test_acc)
logger.debug('Pipe send intermediate result done.')
test_acc = mnist_network.accuracy.eval(
feed_dict={mnist_network.images: mnist.test.images,
mnist_network.labels: mnist.test.labels,
mnist_network.keep_prob: 1.0})
"""@nni.report_final_result(test_acc)"""
logger.debug('Final result is %g', test_acc)
logger.debug('Send final result done.')
def get_params():
''' Get parameters from command line '''
parser = argparse.ArgumentParser()
parser.add_argument("--data_dir", type=str, default='/tmp/tensorflow/mnist/input_data', help="data directory")
parser.add_argument("--dropout_rate", type=float, default=0.5, help="dropout rate")
parser.add_argument("--channel_1_num", type=int, default=32)
parser.add_argument("--channel_2_num", type=int, default=64)
parser.add_argument("--conv_size", type=int, default=5)
parser.add_argument("--pool_size", type=int, default=2)
parser.add_argument("--hidden_size", type=int, default=1024)
parser.add_argument("--learning_rate", type=float, default=1e-4)
parser.add_argument("--batch_num", type=int, default=2000)
parser.add_argument("--batch_size", type=int, default=32)
args, _ = parser.parse_known_args()
return args
if __name__ == '__main__':
try:
params = vars(get_params())
main(params)
except Exception as exception:
logger.exception(exception)
raise
examples/trials/mnist-nas/enas_mode/operators.py deleted 100644 → 0
View file @ b4773e1e
import tensorflow as tf
import math
def weight_variable(shape):
"""weight_variable generates a weight variable of a given shape."""
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial)
def bias_variable(shape):
"""bias_variable generates a bias variable of a given shape."""
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial)
def sum_op(inputs):
"""sum_op"""
fixed_input = inputs[0][0]
optional_input = tf.reduce_sum(inputs[1], axis=0)
if len(optional_input.get_shape()) < 1:
return fixed_input
fixed_shape = fixed_input.get_shape().as_list()
optional_shape = optional_input.get_shape().as_list()
assert fixed_shape[1] == fixed_shape[2]
assert optional_shape[1] == optional_shape[2]
pool_size = math.ceil(optional_shape[1] / fixed_shape[1])
pool_out = tf.nn.avg_pool(optional_input, ksize=[1, pool_size, pool_size, 1], strides=[1, pool_size, pool_size, 1], padding='SAME')
conv_matrix = weight_variable([1, 1, optional_shape[3], fixed_shape[3]])
conv_out = tf.nn.conv2d(pool_out, conv_matrix, strides=[1, 1, 1, 1], padding='SAME')
return fixed_input + conv_out
def conv2d(inputs, size=-1, in_ch=-1, out_ch=-1):
"""conv2d returns a 2d convolution layer with full stride."""
x_input = sum_op(inputs)
if size in [1, 3]:
w_matrix = weight_variable([size, size, in_ch, out_ch])
return tf.nn.conv2d(x_input, w_matrix, strides=[1, 1, 1, 1], padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def twice_conv2d(inputs, size=-1, in_ch=-1, out_ch=-1):
"""twice_conv2d"""
x_input = sum_op(inputs)
if size in [3, 7]:
w_matrix1 = weight_variable([1, size, in_ch, int(out_ch/2)])
out = tf.nn.conv2d(x_input, w_matrix1, strides=[1, 1, 1, 1], padding='SAME')
w_matrix2 = weight_variable([size, 1, int(out_ch/2), out_ch])
return tf.nn.conv2d(out, w_matrix2, strides=[1, 1, 1, 1], padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def dilated_conv(inputs, size=3, in_ch=-1, out_ch=-1):
"""dilated_conv"""
x_input = sum_op(inputs)
if size == 3:
w_matrix = weight_variable([size, size, in_ch, out_ch])
return tf.nn.atrous_conv2d(x_input, w_matrix, rate=2, padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def separable_conv(inputs, size=-1, in_ch=-1, out_ch=-1):
"""separable_conv"""
x_input = sum_op(inputs)
if size in [3, 5, 7]:
depth_matrix = weight_variable([size, size, in_ch, 1])
point_matrix = weight_variable([1, 1, 1*in_ch, out_ch])
return tf.nn.separable_conv2d(x_input, depth_matrix, point_matrix, strides=[1, 1, 1, 1], padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def avg_pool(inputs, size=-1):
"""avg_pool downsamples a feature map."""
x_input = sum_op(inputs)
if size in [3, 5, 7]:
return tf.nn.avg_pool(x_input, ksize=[1, size, size, 1], strides=[1, 1, 1, 1], padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def max_pool(inputs, size=-1):
"""max_pool downsamples a feature map."""
x_input = sum_op(inputs)
if size in [3, 5, 7]:
return tf.nn.max_pool(x_input, ksize=[1, size, size, 1], strides=[1, 1, 1, 1], padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def post_process(inputs, ch_size=-1):
"""post_process"""
x_input = inputs[0][0]
bias_matrix = bias_variable([ch_size])
return tf.nn.relu(x_input + bias_matrix)
examples/trials/mnist-nas/oneshot_mode/config_oneshot.yml deleted 100644 → 0
View file @ b4773e1e
authorName: default
experimentName: example_mnist
trialConcurrency: 1
maxExecDuration: 1h
maxTrialNum: 10
#choice: local, remote, pai
trainingServicePlatform: local
#choice: true, false
useAnnotation: true
tuner:
codeDir: ../../../tuners/random_nas_tuner
classFileName: random_nas_tuner.py
className: RandomNASTuner
trial:
command: python3 mnist-oneshot.py
codeDir: .
gpuNum: 0
nasMode: oneshot_mode
examples/trials/mnist-nas/oneshot_mode/mnist-oneshot.py deleted 100644 → 0
View file @ b4773e1e
"""A deep MNIST classifier using convolutional layers."""
import argparse
import logging
import math
import tempfile
import time
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
import operators as op
FLAGS = None
logger = logging.getLogger('mnist_AutoML')
class MnistNetwork(object):
'''
MnistNetwork is for initializing and building basic network for mnist.
'''
def __init__(self,
channel_1_num,
channel_2_num,
conv_size,
hidden_size,
pool_size,
learning_rate,
x_dim=784,
y_dim=10):
self.channel_1_num = channel_1_num
self.channel_2_num = channel_2_num
self.conv_size = conv_size
self.hidden_size = hidden_size
self.pool_size = pool_size
self.learning_rate = learning_rate
self.x_dim = x_dim
self.y_dim = y_dim
self.images = tf.placeholder(tf.float32, [None, self.x_dim], name='input_x')
self.labels = tf.placeholder(tf.float32, [None, self.y_dim], name='input_y')
self.keep_prob = tf.placeholder(tf.float32, name='keep_prob')
self.train_step = None
self.accuracy = None
def build_network(self):
'''
Building network for mnist, meanwhile specifying its neural architecture search space
'''
# Reshape to use within a convolutional neural net.
# Last dimension is for "features" - there is only one here, since images are
# grayscale -- it would be 3 for an RGB image, 4 for RGBA, etc.
with tf.name_scope('reshape'):
try:
input_dim = int(math.sqrt(self.x_dim))
except:
print(
'input dim cannot be sqrt and reshape. input dim: ' + str(self.x_dim))
logger.debug(
'input dim cannot be sqrt and reshape. input dim: %s', str(self.x_dim))
raise
x_image = tf.reshape(self.images, [-1, input_dim, input_dim, 1])
"""@nni.mutable_layers(
{
layer_choice: [op.conv2d(size=1, in_ch=1, out_ch=self.channel_1_num),
op.conv2d(size=3, in_ch=1, out_ch=self.channel_1_num),
op.twice_conv2d(size=3, in_ch=1, out_ch=self.channel_1_num),
op.twice_conv2d(size=7, in_ch=1, out_ch=self.channel_1_num),
op.dilated_conv(in_ch=1, out_ch=self.channel_1_num),
op.separable_conv(size=3, in_ch=1, out_ch=self.channel_1_num),
op.separable_conv(size=5, in_ch=1, out_ch=self.channel_1_num),
op.separable_conv(size=7, in_ch=1, out_ch=self.channel_1_num)],
fixed_inputs: [x_image],
layer_output: conv1_out
},
{
layer_choice: [op.post_process(ch_size=self.channel_1_num)],
fixed_inputs: [conv1_out],
layer_output: post1_out
},
{
layer_choice: [op.max_pool(size=3),
op.max_pool(size=5),
op.max_pool(size=7),
op.avg_pool(size=3),
op.avg_pool(size=5),
op.avg_pool(size=7)],
fixed_inputs: [post1_out],
layer_output: pool1_out
},
{
layer_choice: [op.conv2d(size=1, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.conv2d(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.twice_conv2d(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.twice_conv2d(size=7, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.dilated_conv(in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.separable_conv(size=3, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.separable_conv(size=5, in_ch=self.channel_1_num, out_ch=self.channel_2_num),
op.separable_conv(size=7, in_ch=self.channel_1_num, out_ch=self.channel_2_num)],
fixed_inputs: [pool1_out],
optional_inputs: [post1_out],
optional_input_size: [0, 1],
layer_output: conv2_out
},
{
layer_choice: [op.post_process(ch_size=self.channel_2_num)],
fixed_inputs: [conv2_out],
layer_output: post2_out
},
{
layer_choice: [op.max_pool(size=3),
op.max_pool(size=5),
op.max_pool(size=7),
op.avg_pool(size=3),
op.avg_pool(size=5),
op.avg_pool(size=7)],
fixed_inputs: [post2_out],
optional_inputs: [post1_out, pool1_out],
optional_input_size: [0, 1],
layer_output: pool2_out
}
)"""
# Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image
# is down to 7x7x64 feature maps -- maps this to 1024 features.
last_dim_list = pool2_out.get_shape().as_list()
assert(last_dim_list[1] == last_dim_list[2])
last_dim = last_dim_list[1]
with tf.name_scope('fc1'):
w_fc1 = op.weight_variable(
[last_dim * last_dim * self.channel_2_num, self.hidden_size])
b_fc1 = op.bias_variable([self.hidden_size])
h_pool2_flat = tf.reshape(
pool2_out, [-1, last_dim * last_dim * self.channel_2_num])
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, w_fc1) + b_fc1)
# Dropout - controls the complexity of the model, prevents co-adaptation of features.
with tf.name_scope('dropout'):
h_fc1_drop = tf.nn.dropout(h_fc1, self.keep_prob)
# Map the 1024 features to 10 classes, one for each digit
with tf.name_scope('fc2'):
w_fc2 = op.weight_variable([self.hidden_size, self.y_dim])
b_fc2 = op.bias_variable([self.y_dim])
y_conv = tf.matmul(h_fc1_drop, w_fc2) + b_fc2
with tf.name_scope('loss'):
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=self.labels, logits=y_conv))
with tf.name_scope('adam_optimizer'):
self.train_step = tf.train.AdamOptimizer(
self.learning_rate).minimize(cross_entropy)
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(
tf.argmax(y_conv, 1), tf.argmax(self.labels, 1))
self.accuracy = tf.reduce_mean(
tf.cast(correct_prediction, tf.float32))
def download_mnist_retry(data_dir, max_num_retries=20):
"""Try to download mnist dataset and avoid errors"""
for _ in range(max_num_retries):
try:
return input_data.read_data_sets(data_dir, one_hot=True)
except tf.errors.AlreadyExistsError:
time.sleep(1)
raise Exception("Failed to download MNIST.")
def main(params):
'''
Main function, build mnist network, run and send result to NNI.
'''
# Import data
mnist = download_mnist_retry(params['data_dir'])
print('Mnist download data done.')
logger.debug('Mnist download data done.')
# Create the model
# Build the graph for the deep net
mnist_network = MnistNetwork(channel_1_num=params['channel_1_num'],
channel_2_num=params['channel_2_num'],
conv_size=params['conv_size'],
hidden_size=params['hidden_size'],
pool_size=params['pool_size'],
learning_rate=params['learning_rate'])
mnist_network.build_network()
logger.debug('Mnist build network done.')
# Write log
graph_location = tempfile.mkdtemp()
logger.debug('Saving graph to: %s', graph_location)
train_writer = tf.summary.FileWriter(graph_location)
train_writer.add_graph(tf.get_default_graph())
test_acc = 0.0
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in range(params['batch_num']):
batch = mnist.train.next_batch(params['batch_size'])
mnist_network.train_step.run(feed_dict={mnist_network.images: batch[0],
mnist_network.labels: batch[1],
mnist_network.keep_prob: 1 - params['dropout_rate']}
)
if i % 100 == 0:
test_acc = mnist_network.accuracy.eval(
feed_dict={mnist_network.images: mnist.test.images,
mnist_network.labels: mnist.test.labels,
mnist_network.keep_prob: 1.0})
"""@nni.report_intermediate_result(test_acc)"""
logger.debug('test accuracy %g', test_acc)
logger.debug('Pipe send intermediate result done.')
test_acc = mnist_network.accuracy.eval(
feed_dict={mnist_network.images: mnist.test.images,
mnist_network.labels: mnist.test.labels,
mnist_network.keep_prob: 1.0})
"""@nni.report_final_result(test_acc)"""
logger.debug('Final result is %g', test_acc)
logger.debug('Send final result done.')
def get_params():
''' Get parameters from command line '''
parser = argparse.ArgumentParser()
parser.add_argument("--data_dir", type=str, default='/tmp/tensorflow/mnist/input_data', help="data directory")
parser.add_argument("--dropout_rate", type=float, default=0.5, help="dropout rate")
parser.add_argument("--channel_1_num", type=int, default=32)
parser.add_argument("--channel_2_num", type=int, default=64)
parser.add_argument("--conv_size", type=int, default=5)
parser.add_argument("--pool_size", type=int, default=2)
parser.add_argument("--hidden_size", type=int, default=1024)
parser.add_argument("--learning_rate", type=float, default=1e-4)
parser.add_argument("--batch_num", type=int, default=2000)
parser.add_argument("--batch_size", type=int, default=32)
args, _ = parser.parse_known_args()
return args
if __name__ == '__main__':
try:
params = vars(get_params())
main(params)
except Exception as exception:
logger.exception(exception)
raise
examples/trials/mnist-nas/oneshot_mode/operators.py deleted 100644 → 0
View file @ b4773e1e
import tensorflow as tf
import math
def weight_variable(shape):
"""weight_variable generates a weight variable of a given shape."""
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial)
def bias_variable(shape):
"""bias_variable generates a bias variable of a given shape."""
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial)
def sum_op(inputs):
"""sum_op"""
fixed_input = inputs[0][0]
optional_input = tf.concat(inputs[1], axis=3)
fixed_shape = fixed_input.get_shape().as_list()
optional_shape = optional_input.get_shape().as_list()
assert fixed_shape[1] == fixed_shape[2]
assert optional_shape[1] == optional_shape[2]
pool_size = math.ceil(optional_shape[1] / fixed_shape[1])
pool_out = tf.nn.avg_pool(optional_input, ksize=[1, pool_size, pool_size, 1], strides=[1, pool_size, pool_size, 1], padding='SAME')
conv_matrix = weight_variable([1, 1, optional_shape[3], fixed_shape[3]])
conv_out = tf.nn.conv2d(pool_out, conv_matrix, strides=[1, 1, 1, 1], padding='SAME')
return fixed_input + conv_out
def conv2d(inputs, size=-1, in_ch=-1, out_ch=-1):
"""conv2d returns a 2d convolution layer with full stride."""
if not inputs[1]:
x_input = inputs[0][0]
else:
x_input = sum_op(inputs)
if size in [1, 3]:
w_matrix = weight_variable([size, size, in_ch, out_ch])
return tf.nn.conv2d(x_input, w_matrix, strides=[1, 1, 1, 1], padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def twice_conv2d(inputs, size=-1, in_ch=-1, out_ch=-1):
"""twice_conv2d"""
if not inputs[1]:
x_input = inputs[0][0]
else:
x_input = sum_op(inputs)
if size in [3, 7]:
w_matrix1 = weight_variable([1, size, in_ch, int(out_ch/2)])
out = tf.nn.conv2d(x_input, w_matrix1, strides=[1, 1, 1, 1], padding='SAME')
w_matrix2 = weight_variable([size, 1, int(out_ch/2), out_ch])
return tf.nn.conv2d(out, w_matrix2, strides=[1, 1, 1, 1], padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def dilated_conv(inputs, size=3, in_ch=-1, out_ch=-1):
"""dilated_conv"""
if not inputs[1]:
x_input = inputs[0][0]
else:
x_input = sum_op(inputs)
if size == 3:
w_matrix = weight_variable([size, size, in_ch, out_ch])
return tf.nn.atrous_conv2d(x_input, w_matrix, rate=2, padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def separable_conv(inputs, size=-1, in_ch=-1, out_ch=-1):
"""separable_conv"""
if not inputs[1]:
x_input = inputs[0][0]
else:
x_input = sum_op(inputs)
if size in [3, 5, 7]:
depth_matrix = weight_variable([size, size, in_ch, 1])
point_matrix = weight_variable([1, 1, 1*in_ch, out_ch])
return tf.nn.separable_conv2d(x_input, depth_matrix, point_matrix, strides=[1, 1, 1, 1], padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def avg_pool(inputs, size=-1):
"""avg_pool downsamples a feature map."""
if not inputs[1]:
x_input = inputs[0][0]
else:
x_input = sum_op(inputs)
if size in [3, 5, 7]:
return tf.nn.avg_pool(x_input, ksize=[1, size, size, 1], strides=[1, 1, 1, 1], padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def max_pool(inputs, size=-1):
"""max_pool downsamples a feature map."""
if not inputs[1]:
x_input = inputs[0][0]
else:
x_input = sum_op(inputs)
if size in [3, 5, 7]:
return tf.nn.max_pool(x_input, ksize=[1, size, size, 1], strides=[1, 1, 1, 1], padding='SAME')
else:
raise Exception("Unknown filter size: %d." % size)
def post_process(inputs, ch_size=-1):
"""post_process"""
x_input = inputs[0][0]
bias_matrix = bias_variable([ch_size])
return tf.nn.relu(x_input + bias_matrix)
examples/trials/nas_cifar10/README.md deleted 100644 → 0
View file @ b4773e1e
# Run Neural Architecture Search in NNI
Now we have an NAS example [NNI-NAS-Example](https://github.com/Crysple/NNI-NAS-Example) run in NNI using NAS interface from our contributors.
We have included its trial code in this folder, and provided example config files to show how to use PPO tuner to tune the trial code.
To prepare for the dataset, please run `cd data && . download.sh`.
Thanks our lovely contributors, and welcome more and more people to join us!
examples/trials/nas_cifar10/README_zh_CN.md deleted 100644 → 0
View file @ b4773e1e
# 在 NNI 中运行神经网络架构搜索
参考 [NNI-NAS-Example](https://github.com/Crysple/NNI-NAS-Example)，来使用贡献者提供的 NAS 接口。
此目录中包含了 Trial 代码，并提供了示例的配置文件来展示如何使用 PPO Tuner 来调优此 Trial 代码。
运行下列代码来准备数据集 `cd data && . download.sh`.
感谢可爱的志愿者，欢迎更多的人加入我们！
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment