Refactor NNI Manager UT (Stage 1) - setup toolchain (#4109)

pipelines/fast-test.yml

@@ -165,6 +165,7 @@ stages:
 
     - script: |
         python setup.py develop
+        mkdir -p coverage
       displayName: Install NNI
 
     - script: |
@@ -185,12 +186,14 @@ stages:
         python -m pytest ut/sdk/test_compressor_tf.py --cov-config=.coveragerc --cov-append
         python -m pytest ut/sdk/test_compressor_torch.py --cov-config=.coveragerc --cov-append
         python -m pytest ut/sdk/test_model_speedup.py --cov-config=.coveragerc --cov-append
+        cp coverage.xml ../coverage/python.xml
       displayName: Python unit test
 
     - script: |
         set -e
         cd ts/nni_manager
         yarn test
+        cp coverage/cobertura-coverage.xml ../../coverage/typescript.xml
       displayName: TypeScript unit test
 
     - task: PublishTestResults@2
@@ -203,7 +206,7 @@ stages:
     - task: PublishCodeCoverageResults@1
       inputs:
         codeCoverageTool: Cobertura
-        summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/*coverage.xml'
+        summaryFileLocation: coverage/*
       displayName: Publish code coverage results
 
     - script: |

ts/nni_manager/.gitignore

 # Build result
-dist/
+/dist
+/.tsbuildinfo
 
 # node modules
-node_modules/
+/node_modules
 
 # test files
 .experiment.test

ts/nni_manager/.mocharc.json

+{
+    "color": true,
+    "require": "test/register.js",
+    "timeout": "15s"
+}

ts/nni_manager/common/component.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
 import * as ioc from 'typescript-ioc';
 
 const Inject: (...args: any[]) => any = ioc.Inject;

ts/nni_manager/common/datastore.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
 import { ExperimentProfile, TrialJobStatistics } from './manager';
 import { TrialJobDetail, TrialJobStatus } from './trainingService';
 

ts/nni_manager/common/errors.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
 export namespace NNIErrorNames {
     export const NOT_FOUND: string = 'NOT_FOUND';
     export const INVALID_JOB_DETAIL: string = 'NO_VALID_JOB_DETAIL_FOUND';

ts/nni_manager/common/experimentConfig.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import * as assert from 'assert';
+import assert from 'assert';
 
 import { KubeflowOperator, OperatorApiVersion } from '../training_service/kubernetes/kubeflow/kubeflowConfig'
 import { KubernetesStorageKind } from '../training_service/kubernetes/kubernetesConfig';

ts/nni_manager/common/experimentManager.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
 abstract class ExperimentManager {
     public abstract getExperimentsInfo(): Promise<JSON>;
     public abstract setExperimentPath(newPath: string): void;

ts/nni_manager/common/experimentStartupInfo.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import * as assert from 'assert';
-import * as os from 'os';
-import * as path from 'path';
+import assert from 'assert';
+import os from 'os';
+import path from 'path';
 
 const API_ROOT_URL: string = '/api/v1/nni';
 

ts/nni_manager/common/log.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import * as fs from 'fs';
+import fs from 'fs';
 import { Writable } from 'stream';
-import * as util from 'util';
+import util from 'util';
 
 /* log level constants */
 

ts/nni_manager/common/manager.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
 import { MetricDataRecord, MetricType, TrialJobInfo } from './datastore';
 import { TrialJobStatus } from './trainingService';
 import { ExperimentConfig } from './experimentConfig';

ts/nni_manager/common/nniConfig.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import * as fs from 'fs';
-import * as path from 'path';
+import fs from 'fs';
+import path from 'path';
 import { promisify } from 'util';
 import { runPythonScript } from './pythonScript';
 

ts/nni_manager/common/observableTimer.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import * as rx from 'rx';
+import rx from 'rx';
 import * as component from '../common/component';
 
 @component.Singleton

ts/nni_manager/common/pythonScript.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
 import { spawn } from 'child_process';
 import { Logger, getLogger } from './log';
 

ts/nni_manager/common/restServer.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import * as assert from 'assert';
-import * as express from 'express';
-import * as http from 'http';
+import assert from 'assert';
+import express from 'express';
+import http from 'http';
 import { Deferred } from 'ts-deferred';
 import { getLogger, Logger } from './log';
 import { getBasePort } from './experimentStartupInfo';

ts/nni_manager/common/shellUtils.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
 // for readability
 const singleQuote = "'";
 const doubleQuote = '"';

ts/nni_manager/common/tensorboardManager.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
 interface TensorboardParams {
     trials: string;
 }

ts/nni_manager/common/trainingService.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
 /**
  * define TrialJobStatus
  */

ts/nni_manager/common/utils.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import * as assert from 'assert';
+import assert from 'assert';
 import { randomBytes } from 'crypto';
-import * as cpp from 'child-process-promise';
-import * as cp from 'child_process';
+import cpp from 'child-process-promise';
+import cp from 'child_process';
 import { ChildProcess, spawn, StdioOptions } from 'child_process';
-import * as dgram from 'dgram';
-import * as fs from 'fs';
-import * as net from 'net';
-import * as os from 'os';
-import * as path from 'path';
+import dgram from 'dgram';
+import fs from 'fs';
+import net from 'net';
+import os from 'os';
+import path from 'path';
 import * as timersPromises from 'timers/promises';
-import * as lockfile from 'lockfile';
+import lockfile from 'lockfile';
 import { Deferred } from 'ts-deferred';
 import { Container } from 'typescript-ioc';
-import * as glob from 'glob';
+import glob from 'glob';
 
 import { Database, DataStore } from './datastore';
 import { getExperimentStartupInfo, setExperimentStartupInfo } from './experimentStartupInfo';
@@ -243,7 +241,7 @@ function countFilesRecursively(directory: string): Promise<number> {
     const deferred: Deferred<number> = new Deferred<number>();
 
     let timeoutId: NodeJS.Timer
-    const delayTimeout: Promise<number> = new Promise((resolve: Function, reject: Function): void => {
+    const delayTimeout: Promise<number> = new Promise((_resolve: Function, reject: Function): void => {
         // Set timeout and reject the promise once reach timeout (5 seconds)
         timeoutId = setTimeout(() => {
             reject(new Error(`Timeout: path ${directory} has too many files`));
@@ -375,11 +373,11 @@ function unixPathJoin(...paths: any[]): string {
  */
 function withLockSync(func: Function, filePath: string, lockOpts: {[key: string]: any}, ...args: any): any {
     const lockName = path.join(path.dirname(filePath), path.basename(filePath) + `.lock.${process.pid}`);
-    if (typeof lockOpts.stale === 'number'){
+    if (typeof lockOpts['stale'] === 'number'){
         const lockPath = path.join(path.dirname(filePath), path.basename(filePath) + '.lock.*');
         const lockFileNames: string[] = glob.sync(lockPath);
         const canLock: boolean = lockFileNames.map((fileName) => {
-            return fs.existsSync(fileName) && Date.now() - fs.statSync(fileName).mtimeMs < lockOpts.stale;
+            return fs.existsSync(fileName) && Date.now() - fs.statSync(fileName).mtimeMs < lockOpts['stale'];
         }).filter(unexpired=>unexpired === true).length === 0;
         if (!canLock) {
             throw new Error('File has been locked.');

ts/nni_manager/core/ipcInterface.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import * as assert from 'assert';
+import assert from 'assert';
 import { ChildProcess } from 'child_process';
 import { EventEmitter } from 'events';
-import * as net from 'net';
+import net from 'net';
 import { Readable, Writable } from 'stream';
 import { NNIError } from '../common/errors';
 import { getLogger, Logger } from '../common/log';