Code coverage report (#559)

* Add UT code coverage report

* updates

* updates

* updates

* updates

* updates

* updates

* integration test python code coverage report

src/nni_manager/core/test/import_all.test.ts

+import * as glob from 'glob';
+
+// Istanbul only generates report for used/imported files, the files are not used/imported by test cases
+// are not included in code coverage reports.
+// This is a workaround to import all files in order to show all source files in code coverage reports.
+
+glob.sync('**/*.ts').forEach((file) => {
+    if (file.indexOf('node_modules/') < 0 && file.indexOf('types/') < 0
+        && file.indexOf('.test.ts') < 0 && file.indexOf('main.ts')) {
+        try {
+            import('../../' + file);
+        } catch(err) {
+        }
+    }
+})

src/nni_manager/package.json

@@ -5,16 +5,19 @@
   "scripts": {
     "postbuild": "cp -rf scripts ./dist/ && cp -rf config ./dist/",
     "build": "tsc",
-    "test": "mocha -r ts-node/register -t 15000 --recursive **/*.test.ts --exclude node_modules/**/**/*.test.ts --exclude core/test/nnimanager.test.ts --colors",
+    "test": "nyc mocha -r ts-node/register -t 15000 --recursive **/*.test.ts --exclude node_modules/**/**/*.test.ts --exclude core/test/nnimanager.test.ts --colors",
     "start": "node dist/main.js",
     "tslint": "tslint -p ."
   },
   "license": "MIT",
   "dependencies": {
+    "azure-storage": "^2.10.2",
     "chai-as-promised": "^7.1.1",
     "child-process-promise": "^2.2.1",
     "express": "^4.16.3",
     "express-joi-validator": "^2.0.0",
+    "js-base64": "^2.4.9",
+    "kubernetes-client": "^6.5.0",
     "node-nvidia-smi": "^1.0.0",
     "rx": "^4.1.0",
     "sqlite3": "^4.0.2",
@@ -25,15 +28,13 @@
     "ts-deferred": "^1.0.4",
     "typescript-ioc": "^1.2.4",
     "typescript-string-operations": "^1.3.1",
-    "webhdfs": "^1.2.0",
-    "azure-storage": "^2.10.2",
-    "kubernetes-client": "^6.5.0",
-    "js-base64": "^2.4.9"
+    "webhdfs": "^1.2.0"
   },
   "devDependencies": {
     "@types/chai": "^4.1.4",
     "@types/chai-as-promised": "^7.1.0",
     "@types/express": "^4.16.0",
+    "@types/glob": "^7.1.1",
     "@types/mocha": "^5.2.5",
     "@types/node": "^10.12.18",
     "@types/request": "^2.47.1",
@@ -43,7 +44,9 @@
     "@types/stream-buffers": "^3.0.2",
     "@types/tmp": "^0.0.33",
     "chai": "^4.1.2",
+    "glob": "^7.1.3",
     "mocha": "^5.2.0",
+    "nyc": "^13.1.0",
     "request": "^2.87.0",
     "rmdir": "^1.2.0",
     "tmp": "^0.0.33",
@@ -54,5 +57,27 @@
   },
   "engines": {
     "node": ">=10.0.0"
+  },
+  "nyc": {
+    "include": [
+      "**/*.ts"
+    ],
+    "exclude": [
+      "**/test/*",
+      "./node_modules/"
+    ],
+    "extension": [
+      ".ts",
+      ".tsx"
+    ],
+    "require": [
+      "ts-node/register"
+    ],
+    "reporter": [
+      "text-summary",
+      "html"
+    ],
+    "sourceMap": true,
+    "instrument": true
   }
 }

src/sdk/pynni/nni/__main__.py

@@ -34,6 +34,10 @@ from nni.multi_phase.multi_phase_dispatcher import MultiPhaseMsgDispatcher
 logger = logging.getLogger('nni.main')
 logger.debug('START')
 
+if os.environ.get('COVERAGE_PROCESS_START'):
+    import coverage
+    coverage.process_startup()
+
 def augment_classargs(input_class_args, classname):
     if classname in ClassArgs:
         for key, value in ClassArgs[classname].items():

src/sdk/pynni/nni/msg_dispatcher_base.py

@@ -46,7 +46,7 @@ class MsgDispatcherBase(Recoverable):
         while True:
             _logger.debug('waiting receive_message')
             command, data = receive()
-            if command is None:
+            if command is None or command is CommandType.Terminate:
                 break
             if multi_thread_enabled():
                 self.pool.map_async(self.handle_request, [(command, data)])
@@ -64,11 +64,6 @@ class MsgDispatcherBase(Recoverable):
 
         _logger.debug('handle request: command: [{}], data: [{}]'.format(command, data))
 
-        if command is CommandType.Terminate:
-            # if receive Terminate command, exit process
-            _logger.info('Receive Terminate command from NNI manager, terminating')
-            exit(0)
-
         data = json_tricks.loads(data)
 
         command_handlers = {

src/sdk/pynni/setup.py

@@ -35,7 +35,8 @@ setuptools.setup(
         'hyperopt',
         'json_tricks',
         'numpy',
-        'scipy'
+        'scipy',
+        'coverage'
     ],
     package_data = {'nni': ['**/requirements.txt']},
 

src/sdk/pynni/ut.sh

+#!/bin/bash
+coverage run setup.py test
+coverage html

test/.coveragerc

+# .coveragerc to control coverage.py
+[run]
+branch = True
+parallel = True
+data_file = ${COVERAGE_DATA_FILE}
+source = nni, nni_cmd, nni_trial_tool
+
+concurrency = multiprocessing
+
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain about missing debug-only code:
+    def __repr__
+    if self\.debug
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+
+    # Don't complain if non-runnable code isn't run:
+    if 0:
+    if __name__ == .__main__.:
+
+ignore_errors = True
+
+[html]
+directory = ${COVERAGE_HTML_DIR}

test/it.sh

+#!/bin/bash
+CWD=${PWD}
+
+## Export certain environment variables for unittest code to work
+export COVERAGE_PROCESS_START=${CWD}/.coveragerc
+export COVERAGE_DATA_FILE=${CWD}/coverage/data
+export COVERAGE_HTML_DIR=${CWD}/coverhtml
+
+rm ${COVERAGE_DATA_FILE}*
+rm -rf ${COVERAGE_HTML_DIR}/*
+mkdir ${CWD}/coverage
+mkdir ${COVERAGE_HTML_DIR}
+
+## ------Run integration test------
+echo "===========================Testing: integration test==========================="
+coverage run sdk_test.py
+coverage combine
+coverage html

tools/nni_cmd/nnictl.py

@@ -28,6 +28,10 @@ from .package_management import *
 from .constants import *
 from .tensorboard_utils import *
 
+if os.environ.get('COVERAGE_PROCESS_START'):
+    import coverage
+    coverage.process_startup()
+
 def nni_info(*args):
     if args[0].version:
         print(pkg_resources.get_distribution('nni').version)