Unify nas searchspace (#1393)

* unify NAS search space, update random nas tuner and docs

docs/en_US/Tutorial/SearchSpaceSpec.md

@@ -72,6 +72,11 @@ All types of sampling strategies and their parameter are listed here:
   * Which means the variable value is a value like round(exp(normal(mu, sigma)) / q) * q
   * Suitable for a discrete variable with respect to which the objective is smooth and gets smoother with the size of the variable, which is bounded from one side.
 
+* {"_type":"mutable_layer","_value":{mutable_layer_infomation}}
+  * Type for [Neural Architecture Search Space][1]. Value is also a dictionary, which contains key-value pairs representing respectively name and search space of each mutable_layer.
+  * For now, users can only use this type of search space with annotation, which means that there is no need to define a json file for search space since it will be automatically generated according to the annotation in trial code.
+  * For detailed usage, please refer to [General NAS Interfaces][1].
+
 ## Search Space Types Supported by Each Tuner
 
 |                   | choice  | randint | uniform | quniform | loguniform | qloguniform | normal  | qnormal | lognormal | qlognormal |
@@ -103,3 +108,5 @@ Known Limitations:
     * Only Random Search/TPE/Anneal/Evolution tuner supports nested search space
 
     * We do not support nested search space "Hyper Parameter" in visualization now, the enhancement is being considered in #1110(https://github.com/microsoft/nni/issues/1110), any suggestions or discussions or contributions are warmly welcomed
+
+[1]: ../AdvancedFeature/GeneralNasInterfaces.md
\ No newline at end of file

examples/tuners/random_nas_tuner/random_nas_tuner.py

@@ -7,7 +7,9 @@ def random_archi_generator(nas_ss, random_state):
     '''
     chosen_archi = {}
     print("zql: nas search space: ", nas_ss)
-    for block_name, block in nas_ss.items():
+    for block_name, block_value in nas_ss.items():
+        assert block_value['_type'] == "mutable_layer", "Random NAS Tuner only receives NAS search space whose _type is 'mutable_layer'"
+        block = block_value['_value']
         tmp_block = {}
         for layer_name, layer in block.items():
             tmp_layer = {}

tools/nni_annotation/search_space_generator.py

@@ -57,8 +57,8 @@ class SearchSpaceGenerator(ast.NodeTransformer):
         key = self.module_name + '/' + mutable_block
         args[0].s = key
         if key not in self.search_space:
-            self.search_space[key] = dict()
-        self.search_space[key][mutable_layer] = {
+            self.search_space[key] = {'_type': 'mutable_layer', '_value': {}}
+        self.search_space[key]['_value'][mutable_layer] = {
             'layer_choice': [k.s for k in args[2].keys],
             'optional_inputs': [k.s for k in args[5].keys],
             'optional_input_size': args[6].n if isinstance(args[6], ast.Num) else [args[6].elts[0].n, args[6].elts[1].n]

tools/nni_annotation/test_annotation.py

@@ -44,8 +44,9 @@ class AnnotationTestCase(TestCase):
             self.assertEqual(search_space, json.load(f))
 
     def test_code_generator(self):
-        code_dir = expand_annotations('testcase/usercode', '_generated')
+        code_dir = expand_annotations('testcase/usercode', '_generated', nas_mode='classic_mode')
         self.assertEqual(code_dir, '_generated')
+        self._assert_source_equal('testcase/annotated/nas.py', '_generated/nas.py')
         self._assert_source_equal('testcase/annotated/mnist.py', '_generated/mnist.py')
         self._assert_source_equal('testcase/annotated/dir/simple.py', '_generated/dir/simple.py')
         with open('testcase/usercode/nonpy.txt') as src, open('_generated/nonpy.txt') as dst:

tools/nni_annotation/testcase/annotated/nas.py

+import nni
+import time
+
+
+def add_one(inputs):
+    return inputs + 1
+
+
+def add_two(inputs):
+    return inputs + 2
+
+
+def add_three(inputs):
+    return inputs + 3
+
+
+def add_four(inputs):
+    return inputs + 4
+
+
+def main():
+    images = 5
+    layer_1_out = nni.mutable_layer('mutable_block_39', 'mutable_layer_0',
+        {'add_one()': add_one, 'add_two()': add_two, 'add_three()':
+        add_three, 'add_four()': add_four}, {'add_one()': {}, 'add_two()':
+        {}, 'add_three()': {}, 'add_four()': {}}, [], {'images': images}, 1,
+        'classic_mode')
+    layer_2_out = nni.mutable_layer('mutable_block_39', 'mutable_layer_1',
+        {'add_one()': add_one, 'add_two()': add_two, 'add_three()':
+        add_three, 'add_four()': add_four}, {'add_one()': {}, 'add_two()':
+        {}, 'add_three()': {}, 'add_four()': {}}, [], {'layer_1_out':
+        layer_1_out}, 1, 'classic_mode')
+    layer_3_out = nni.mutable_layer('mutable_block_39', 'mutable_layer_2',
+        {'add_one()': add_one, 'add_two()': add_two, 'add_three()':
+        add_three, 'add_four()': add_four}, {'add_one()': {}, 'add_two()':
+        {}, 'add_three()': {}, 'add_four()': {}}, [], {'layer_1_out':
+        layer_1_out, 'layer_2_out': layer_2_out}, 1, 'classic_mode')
+    nni.report_intermediate_result(layer_1_out)
+    time.sleep(2)
+    nni.report_intermediate_result(layer_2_out)
+    time.sleep(2)
+    nni.report_intermediate_result(layer_3_out)
+    time.sleep(2)
+    layer_3_out = layer_3_out + 10
+    nni.report_final_result(layer_3_out)
+
+
+if __name__ == '__main__':
+    main()

tools/nni_annotation/testcase/searchspace.json

@@ -143,5 +143,47 @@
             "(2 * 3 + 4)",
             "(lambda x: 1 + x)"
         ]
+    },
+    "nas/mutable_block_39": {
+        "_type": "mutable_layer",
+        "_value": {
+            "mutable_layer_0": {
+                "layer_choice": [
+                    "add_one()",
+                    "add_two()",
+                    "add_three()",
+                    "add_four()"
+                ],
+                "optional_inputs": [
+                    "images"
+                ],
+                "optional_input_size": 1
+            },
+            "mutable_layer_1": {
+                "layer_choice": [
+                    "add_one()",
+                    "add_two()",
+                    "add_three()",
+                    "add_four()"
+                ],
+                "optional_inputs": [
+                    "layer_1_out"
+                ],
+                "optional_input_size": 1
+            },
+            "mutable_layer_2": {
+                "layer_choice": [
+                    "add_one()",
+                    "add_two()",
+                    "add_three()",
+                    "add_four()"
+                ],
+                "optional_inputs": [
+                    "layer_1_out",
+                    "layer_2_out"
+                ],
+                "optional_input_size": 1
+            }
+        }
     }
 }
\ No newline at end of file