Python wrapper for rest api (#1318)

Makefile

@@ -167,6 +167,7 @@ dev-install-python-modules:
 	#$(_INFO) Installing Python SDK $(_END)
 	mkdir -p build
 	ln -sf ../src/sdk/pynni/nni build/nni
+	ln -sf ../src/sdk/pynni/nnicli build/nnicli
 	ln -sf ../tools/nni_annotation build/nni_annotation
 	ln -sf ../tools/nni_cmd build/nni_cmd
 	ln -sf ../tools/nni_trial_tool build/nni_trial_tool

azure-pipelines.yml

@@ -34,6 +34,10 @@ jobs:
       cd test
       PATH=$HOME/.local/bin:$PATH python3 metrics_test.py
     displayName: 'Trial job metrics test'
+  - script: |
+      cd test
+      PATH=$HOME/.local/bin:$PATH python3 cli_test.py
+    displayName: 'nnicli test'
 
 - job: 'basic_test_pr_macOS'
   pool:
@@ -61,3 +65,7 @@ jobs:
       cd test
       PATH=$HOME/Library/Python/3.7/bin:$PATH python3 tuner_test.py
     displayName: 'Built-in tuners / assessors tests'
+  - script: |
+      cd test
+      PATH=$HOME/Library/Python/3.7/bin:$PATH python3 cli_test.py
+    displayName: 'nnicli test'

deployment/pypi/setup.py

@@ -53,13 +53,16 @@ setuptools.setup(
     long_description_content_type = 'text/markdown',
     license = 'MIT',
     url = 'https://github.com/Microsoft/nni',
-    packages = setuptools.find_packages('../../tools') + setuptools.find_packages('../../src/sdk/pynni', exclude=['tests']),
+    packages = setuptools.find_packages('../../tools') \
+        + setuptools.find_packages('../../src/sdk/pynni', exclude=['tests']) \
+        + setuptools.find_packages('../../src/sdk/pycli'),
     package_dir = {
         'nni_annotation': '../../tools/nni_annotation',
         'nni_cmd': '../../tools/nni_cmd',
         'nni_trial_tool': '../../tools/nni_trial_tool',
         'nni_gpu_tool': '../../tools/nni_gpu_tool',
-        'nni': '../../src/sdk/pynni/nni'
+        'nni': '../../src/sdk/pynni/nni',
+        'nnicli': '../../src/sdk/pycli/nnicli'
     },
     package_data = {'nni': ['**/requirements.txt']},
     python_requires = '>=3.5',

setup.py

@@ -35,9 +35,10 @@ setup(
     license = 'MIT',
     url = 'https://github.com/Microsoft/nni',
 
-    packages = find_packages('src/sdk/pynni', exclude=['tests']) + find_packages('tools'),
+    packages = find_packages('src/sdk/pynni', exclude=['tests']) + find_packages('src/sdk/pycli') + find_packages('tools'),
     package_dir = {
         'nni': 'src/sdk/pynni/nni',
+        'nnicli': 'src/sdk/pycli/nnicli',
         'nni_annotation': 'tools/nni_annotation',
         'nni_cmd': 'tools/nni_cmd',
         'nni_trial_tool':'tools/nni_trial_tool',

src/sdk/pycli/nnicli/__init__.py

+# Copyright (c) Microsoft Corporation. All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+# associated documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish, distribute,
+# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# ==================================================================================================
+
+from .nni_client import *

src/sdk/pycli/nnicli/nni_client.py

+# Copyright (c) Microsoft Corporation. All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+# associated documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish, distribute,
+# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# ==================================================================================================
+
+""" A python wrapper for nni rest api
+
+Example:
+
+import nnicli as nc
+
+nc.start_nni('../../../../examples/trials/mnist/config.yml')
+
+nc.set_endpoint('http://localhost:8080')
+
+print(nc.version())
+print(nc.get_experiment_status())
+
+print(nc.get_job_statistics())
+print(nc.list_trial_jobs())
+
+nc.stop_nni()
+
+"""
+
+import sys
+import os
+import subprocess
+import requests
+
+__all__ = [
+    'start_nni',
+    'stop_nni',
+    'set_endpoint',
+    'version',
+    'get_experiment_status',
+    'get_experiment_profile',
+    'get_trial_job',
+    'list_trial_jobs',
+    'get_job_statistics',
+    'get_job_metrics',
+    'export_data'
+]
+
+EXPERIMENT_PATH = 'experiment'
+VERSION_PATH = 'version'
+STATUS_PATH = 'check-status'
+JOB_STATISTICS_PATH = 'job-statistics'
+TRIAL_JOBS_PATH = 'trial-jobs'
+METRICS_PATH = 'metric-data'
+EXPORT_DATA_PATH = 'export-data'
+
+API_ROOT_PATH = 'api/v1/nni'
+
+_api_endpoint = None
+
+def set_endpoint(endpoint):
+    """set endpoint of nni rest server for nnicli, for example:
+    http://localhost:8080
+    """
+    global _api_endpoint
+    _api_endpoint = endpoint
+
+def _check_endpoint():
+    if _api_endpoint is None:
+        raise AssertionError("Please call set_endpoint to specify nni endpoint")
+
+def _nni_rest_get(api_path, response_type='json'):
+    _check_endpoint()
+    uri = '{}/{}/{}'.format(_api_endpoint, API_ROOT_PATH, api_path)
+    res = requests.get(uri)
+    if _http_succeed(res.status_code):
+        if response_type == 'json':
+            return res.json()
+        elif response_type == 'text':
+            return res.text
+        else:
+            raise AssertionError('Incorrect response_type')
+    else:
+        return None
+
+def _http_succeed(status_code):
+    return status_code // 100 == 2
+
+def _create_process(cmd):
+    if sys.platform == 'win32':
+        process = subprocess.Popen(cmd, stdout=subprocess.PIPE, creationflags=subprocess.CREATE_NEW_PROCESS_GROUP)
+    else:
+        process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+
+    while process.poll() is None:
+        output = process.stdout.readline()
+        if output:
+            print(output.decode('utf-8').strip())
+    return process.returncode
+
+def start_nni(config_file):
+    """start nni experiment with specified configuration file"""
+    cmd = 'nnictl create --config {}'.format(config_file).split(' ')
+    if _create_process(cmd) != 0:
+        raise RuntimeError('Failed to start nni.')
+
+def stop_nni():
+    """stop nni experiment"""
+    cmd = 'nnictl stop'.split(' ')
+    if _create_process(cmd) != 0:
+        raise RuntimeError('Failed to stop nni.')
+
+def version():
+    """return version of nni"""
+    return _nni_rest_get(VERSION_PATH, 'text')
+
+def get_experiment_status():
+    """return experiment status as a dict"""
+    return _nni_rest_get(STATUS_PATH)
+
+def get_experiment_profile():
+    """return experiment profile as a dict"""
+    return _nni_rest_get(EXPERIMENT_PATH)
+
+def get_trial_job(trial_job_id):
+    """return trial job information as a dict"""
+    assert trial_job_id is not None
+    return _nni_rest_get(os.path.join(TRIAL_JOBS_PATH, trial_job_id))
+
+def list_trial_jobs():
+    """return information for all trial jobs as a list"""
+    return _nni_rest_get(TRIAL_JOBS_PATH)
+
+def get_job_statistics():
+    """return trial job statistics information as a dict"""
+    return _nni_rest_get(JOB_STATISTICS_PATH)
+
+def get_job_metrics(trial_job_id=None):
+    """return trial job metrics"""
+    api_path = METRICS_PATH if trial_job_id is None else os.path.join(METRICS_PATH, trial_job_id)
+    return _nni_rest_get(api_path)
+
+def export_data():
+    """return exported information for all trial jobs"""
+    return _nni_rest_get(EXPORT_DATA_PATH)

src/sdk/pycli/setup.py

+import setuptools
+
+setuptools.setup(
+    name = 'nnicli',
+    version = '999.0.0-developing',
+    packages = setuptools.find_packages(),
+
+    python_requires = '>=3.5',
+    install_requires = [
+        'requests'
+    ],
+
+    author = 'Microsoft NNI Team',
+    author_email = 'nni@microsoft.com',
+    description = 'nnicli for Neural Network Intelligence project',
+    license = 'MIT',
+    url = 'https://github.com/Microsoft/nni',
+)

test/cli_test.py

+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge,
+# to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import sys
+import time
+import traceback
+from utils import GREEN, RED, CLEAR, setup_experiment
+
+def test_nni_cli():
+    import nnicli as nc
+
+    config_file = 'config_test/examples/mnist.test.yml'
+
+    try:
+        # Sleep here to make sure previous stopped exp has enough time to exit to avoid port conflict
+        time.sleep(6)
+        print(GREEN + 'Testing nnicli:' + config_file + CLEAR)
+        nc.start_nni(config_file)
+        time.sleep(3)
+        nc.set_endpoint('http://localhost:8080')
+        print(nc.version())
+        print(nc.get_job_statistics())
+        print(nc.get_experiment_status())
+        nc.list_trial_jobs()
+
+        print(GREEN + 'Test nnicli {}: TEST PASS'.format(config_file) + CLEAR)
+    except Exception as error:
+        print(RED + 'Test nnicli {}: TEST FAIL'.format(config_file) + CLEAR)
+        print('%r' % error)
+        traceback.print_exc()
+        raise error
+    finally:
+        nc.stop_nni()
+
+if __name__ == '__main__':
+    installed = (sys.argv[-1] != '--preinstall')
+    setup_experiment(installed)
+
+    test_nni_cli()

test/pipelines-it-local-windows.yml

@@ -36,3 +36,7 @@ jobs:
       cd test
       python metrics_test.py
     displayName: 'Trial job metrics test'
+  - script: |
+      cd test
+      PATH=$HOME/.local/bin:$PATH python3 cli_test.py
+    displayName: 'nnicli test'

test/pipelines-it-local.yml

@@ -37,3 +37,7 @@ jobs:
       cd test
       PATH=$HOME/.local/bin:$PATH python3 metrics_test.py
     displayName: 'Trial job metrics test'
+  - script: |
+      cd test
+      PATH=$HOME/.local/bin:$PATH python3 cli_test.py
+    displayName: 'nnicli test'