Refactor NNI Manager UT (Stage 1) - setup toolchain (#4109)

ts/nni_manager/training_service/reusable/environment.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import { getLogger, Logger } from "../../common/log";
-import { TrialJobStatus } from "../../common/trainingService";
-import { GPUInfo } from "../../training_service/common/gpuData";
+import { getLogger, Logger } from "common/log";
+import { TrialJobStatus } from "common/trainingService";
+import { GPUInfo } from "training_service/common/gpuData";
 import { CommandChannel } from "./commandChannel";
 import { WebCommandChannel } from './channels/webCommandChannel';
 import { EventEmitter } from "events";

ts/nni_manager/training_service/reusable/environments/amlEnvironmentService.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import * as fs from 'fs';
-import * as path from 'path';
-import * as component from '../../../common/component';
-import { getLogger, Logger } from '../../../common/log';
-import { ExperimentConfig, AmlConfig, flattenConfig } from '../../../common/experimentConfig';
-import { ExperimentStartupInfo } from '../../../common/experimentStartupInfo';
-import { validateCodeDir } from '../../common/util';
+import fs from 'fs';
+import path from 'path';
+import * as component from 'common/component';
+import { getLogger, Logger } from 'common/log';
+import { ExperimentConfig, AmlConfig, flattenConfig } from 'common/experimentConfig';
+import { ExperimentStartupInfo } from 'common/experimentStartupInfo';
+import { validateCodeDir } from 'training_service/common/util';
 import { AMLClient } from '../aml/amlClient';
 import { AMLEnvironmentInformation } from '../aml/amlConfig';
 import { EnvironmentInformation, EnvironmentService } from '../environment';

ts/nni_manager/training_service/reusable/environments/dlcEnvironmentService.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import * as fs from 'fs';
-import * as path from 'path';
-import * as component from '../../../common/component';
-import { getLogger, Logger } from '../../../common/log';
-import { ExperimentConfig, DlcConfig, flattenConfig } from '../../../common/experimentConfig';
-import { ExperimentStartupInfo } from '../../../common/experimentStartupInfo';
+import fs from 'fs';
+import path from 'path';
+import * as component from 'common/component';
+import { getLogger, Logger } from 'common/log';
+import { ExperimentConfig, DlcConfig, flattenConfig } from 'common/experimentConfig';
+import { ExperimentStartupInfo } from 'common/experimentStartupInfo';
 import { DlcClient } from '../dlc/dlcClient';
 import { DlcEnvironmentInformation } from '../dlc/dlcConfig';
 import { EnvironmentInformation, EnvironmentService } from '../environment';

ts/nni_manager/training_service/reusable/environments/environmentServiceFactory.ts

@@ -4,10 +4,10 @@ import { LocalEnvironmentService } from './localEnvironmentService';
 import { RemoteEnvironmentService } from './remoteEnvironmentService';
 import { KubeflowEnvironmentService } from './kubernetes/kubeflowEnvironmentService';
 import { EnvironmentService } from '../environment';
-import { ExperimentConfig } from '../../../common/experimentConfig';
-import { ExperimentStartupInfo } from '../../../common/experimentStartupInfo';
-import { getCustomEnvironmentServiceConfig } from '../../../common/nniConfig';
-import { importModule } from '../../../common/utils';
+import { ExperimentConfig } from 'common/experimentConfig';
+import { ExperimentStartupInfo } from 'common/experimentStartupInfo';
+import { getCustomEnvironmentServiceConfig } from 'common/nniConfig';
+import { importModule } from 'common/utils';
 import { DlcEnvironmentService } from './dlcEnvironmentService';
 
 export async function createEnvironmentService(name: string, config: ExperimentConfig): Promise<EnvironmentService> {

ts/nni_manager/training_service/reusable/environments/kubernetes/kubeflowEnvironmentService.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import * as fs from 'fs';
-import * as path from 'path';
-import * as component from '../../../../common/component';
-import { ExperimentConfig, KubeflowConfig, flattenConfig } from '../../../../common/experimentConfig';
-import { ExperimentStartupInfo } from '../../../../common/experimentStartupInfo';
-import { EnvironmentInformation } from '../../environment';
+import fs from 'fs';
+import path from 'path';
+import * as component from 'common/component';
+import { ExperimentConfig, KubeflowConfig, flattenConfig } from 'common/experimentConfig';
+import { ExperimentStartupInfo } from 'common/experimentStartupInfo';
+import { EnvironmentInformation } from 'training_service/reusable/environment';
 import { KubernetesEnvironmentService } from './kubernetesEnvironmentService';
-import { KubeflowOperatorClientFactory } from '../../../kubernetes/kubeflow/kubeflowApiClient';
-import { KubeflowClusterConfigAzure } from '../../../kubernetes/kubeflow/kubeflowConfig';
-import { KeyVaultConfig, AzureStorage } from '../../../kubernetes/kubernetesConfig';
+import { KubeflowOperatorClientFactory } from 'training_service/kubernetes/kubeflow/kubeflowApiClient';
+import { KubeflowClusterConfigAzure } from 'training_service/kubernetes/kubeflow/kubeflowConfig';
+import { KeyVaultConfig, AzureStorage } from 'training_service/kubernetes/kubernetesConfig';
 
 interface FlattenKubeflowConfig extends ExperimentConfig, KubeflowConfig { }
 

ts/nni_manager/training_service/reusable/environments/kubernetes/kubernetesEnvironmentService.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import * as cpp from 'child-process-promise';
-import * as path from 'path';
-import * as azureStorage from 'azure-storage';
+import cpp from 'child-process-promise';
+import path from 'path';
+import azureStorage from 'azure-storage';
 import {Base64} from 'js-base64';
 import {String} from 'typescript-string-operations';
-import { ExperimentConfig } from '../../../../common/experimentConfig';
-import { ExperimentStartupInfo } from '../../../../common/experimentStartupInfo';
-import { getLogger, Logger } from '../../../../common/log';
-import { EnvironmentInformation, EnvironmentService } from '../../environment';
-import {GeneralK8sClient, KubernetesCRDClient} from '../../../kubernetes/kubernetesApiClient';
-import {AzureStorageClientUtility} from '../../../kubernetes/azureStorageClientUtils';
-import { KubeflowJobStatus } from '../../../kubernetes/kubeflow/kubeflowConfig';
-import {delay, uniqueString} from '../../../../common/utils';
+import { ExperimentConfig } from 'common/experimentConfig';
+import { ExperimentStartupInfo } from 'common/experimentStartupInfo';
+import { getLogger, Logger } from 'common/log';
+import { EnvironmentInformation, EnvironmentService } from 'training_service/reusable/environment';
+import {GeneralK8sClient, KubernetesCRDClient} from 'training_service/kubernetes/kubernetesApiClient';
+import {AzureStorageClientUtility} from 'training_service/kubernetes/azureStorageClientUtils';
+import { KubeflowJobStatus } from 'training_service/kubernetes/kubeflow/kubeflowConfig';
+import {delay, uniqueString} from 'common/utils';
 const fs = require('fs');
 
 export class KubernetesEnvironmentService extends EnvironmentService {
@@ -35,7 +33,7 @@ export class KubernetesEnvironmentService extends EnvironmentService {
     protected log: Logger = getLogger('KubernetesEnvironmentService');
     protected environmentWorkingFolder: string;
 
-    constructor(config: ExperimentConfig, info: ExperimentStartupInfo) {
+    constructor(_config: ExperimentConfig, info: ExperimentStartupInfo) {
         super();
         this.CONTAINER_MOUNT_PATH = '/tmp/mount';
         this.genericK8sClient = new GeneralK8sClient();
@@ -234,7 +232,7 @@ export class KubernetesEnvironmentService extends EnvironmentService {
         });
     }
 
-    public async startEnvironment(environment: EnvironmentInformation): Promise<void> {
+    public async startEnvironment(_environment: EnvironmentInformation): Promise<void> {
         throw new Error("Not implemented");
     }
 

ts/nni_manager/training_service/reusable/environments/localEnvironmentService.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import * as fs from 'fs';
-import * as path from 'path';
-import * as tkill from 'tree-kill';
-import * as component from '../../../common/component';
-import { getLogger, Logger } from '../../../common/log';
-import { ExperimentConfig } from '../../../common/experimentConfig';
-import { ExperimentStartupInfo } from '../../../common/experimentStartupInfo';
-import { powershellString } from '../../../common/shellUtils';
+import fs from 'fs';
+import path from 'path';
+import tkill from 'tree-kill';
+import * as component from 'common/component';
+import { getLogger, Logger } from 'common/log';
+import { ExperimentConfig } from 'common/experimentConfig';
+import { ExperimentStartupInfo } from 'common/experimentStartupInfo';
+import { powershellString } from 'common/shellUtils';
 import { EnvironmentInformation, EnvironmentService } from '../environment';
-import { isAlive, getNewLine } from '../../../common/utils';
-import { execMkdir, runScript, getScriptName, execCopydir } from '../../common/util';
+import { isAlive, getNewLine } from 'common/utils';
+import { execMkdir, runScript, getScriptName, execCopydir } from 'training_service/common/util';
 import { SharedStorageService } from '../sharedStorage'
 
 @component.Singleton
@@ -81,7 +79,7 @@ export class LocalEnvironmentService extends EnvironmentService {
     private getScript(environment: EnvironmentInformation): string[] {
         const script: string[] = [];
         if (process.platform === 'win32') {
-            script.push(`$env:PATH=${powershellString(process.env.path!)}`)
+            script.push(`$env:PATH=${powershellString(process.env['path']!)}`)
             script.push(`cd $env:${this.experimentRootDir}`);
             script.push(`New-Item -ItemType "directory" -Path ${path.join(this.experimentRootDir, 'envs', environment.id)} -Force`);
             script.push(`cd envs\\${environment.id}`);

ts/nni_manager/training_service/reusable/environments/openPaiEnvironmentService.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import * as yaml from 'js-yaml';
-import * as request from 'request';
+import yaml from 'js-yaml';
+import request from 'request';
 import { Container, Scope } from 'typescript-ioc';
 import { Deferred } from 'ts-deferred';
-import * as component from '../../../common/component';
-import { ExperimentConfig, OpenpaiConfig, flattenConfig, toMegaBytes } from '../../../common/experimentConfig';
-import { ExperimentStartupInfo } from '../../../common/experimentStartupInfo';
-import { getLogger, Logger } from '../../../common/log';
-import { PAIClusterConfig } from '../../pai/paiConfig';
-import { NNIPAITrialConfig } from '../../pai/paiConfig';
+import * as component from 'common/component';
+import { ExperimentConfig, OpenpaiConfig, flattenConfig, toMegaBytes } from 'common/experimentConfig';
+import { ExperimentStartupInfo } from 'common/experimentStartupInfo';
+import { getLogger, Logger } from 'common/log';
+import { PAIClusterConfig } from 'training_service/pai/paiConfig';
+import { NNIPAITrialConfig } from 'training_service/pai/paiConfig';
 import { EnvironmentInformation, EnvironmentService } from '../environment';
 import { SharedStorageService } from '../sharedStorage';
 import { MountedStorageService } from '../storages/mountedStorageService';

ts/nni_manager/training_service/reusable/environments/remoteEnvironmentService.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import * as fs from 'fs';
-import * as path from 'path';
-import * as component from '../../../common/component';
-import { getLogger, Logger } from '../../../common/log';
+import fs from 'fs';
+import path from 'path';
+import * as component from 'common/component';
+import { getLogger, Logger } from 'common/log';
 import { EnvironmentInformation, EnvironmentService } from '../environment';
-import { getLogLevel } from '../../../common/utils';
-import { ExperimentConfig, RemoteConfig, RemoteMachineConfig, flattenConfig } from '../../../common/experimentConfig';
-import { ExperimentStartupInfo } from '../../../common/experimentStartupInfo';
-import { execMkdir } from '../../common/util';
-import { ExecutorManager } from '../../remote_machine/remoteMachineData';
+import { getLogLevel } from 'common/utils';
+import { ExperimentConfig, RemoteConfig, RemoteMachineConfig, flattenConfig } from 'common/experimentConfig';
+import { ExperimentStartupInfo } from 'common/experimentStartupInfo';
+import { execMkdir } from 'training_service/common/util';
+import { ExecutorManager } from 'training_service/remote_machine/remoteMachineData';
 import { ShellExecutor } from 'training_service/remote_machine/shellExecutor';
 import { RemoteMachineEnvironmentInformation } from '../remote/remoteConfig';
 import { SharedStorageService } from '../sharedStorage'

ts/nni_manager/training_service/reusable/gpuScheduler.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import * as assert from 'assert';
+import assert from 'assert';
 import { PlacementConstraint } from 'common/trainingService';
-import { getLogger, Logger } from '../../common/log';
-import { randomSelect } from '../../common/utils';
+import { getLogger, Logger } from 'common/log';
+import { randomSelect } from 'common/utils';
 import { GPUInfo, ScheduleResultType } from '../common/gpuData';
 import { EnvironmentInformation } from './environment';
 import { RemoteMachineEnvironmentInformation } from './remote/remoteConfig';
@@ -232,6 +230,7 @@ export class GpuScheduler {
                 assert(false, 'gpuInfos is undefined');
             }
         }
+        return undefined;
     }
 
     /**

ts/nni_manager/training_service/reusable/remote/remoteConfig.ts

@@ -2,7 +2,7 @@
 // Licensed under the MIT license.
 
 import { EnvironmentInformation } from '../environment';
-import { RemoteMachineConfig } from '../../../common/experimentConfig';
+import { RemoteMachineConfig } from 'common/experimentConfig';
 
 /**
  * RemoteMachineEnvironmentInformation

ts/nni_manager/training_service/reusable/routerTrainingService.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import { getLogger, Logger } from '../../common/log';
-import { MethodNotImplementedError } from '../../common/errors';
-import { ExperimentConfig, RemoteConfig, OpenpaiConfig, KubeflowConfig } from '../../common/experimentConfig';
-import { TrainingService, TrialJobApplicationForm, TrialJobDetail, TrialJobMetric } from '../../common/trainingService';
-import { delay } from '../../common/utils';
+import { getLogger, Logger } from 'common/log';
+import { MethodNotImplementedError } from 'common/errors';
+import { ExperimentConfig, RemoteConfig, OpenpaiConfig, KubeflowConfig } from 'common/experimentConfig';
+import { TrainingService, TrialJobApplicationForm, TrialJobDetail, TrialJobMetric } from 'common/trainingService';
+import { delay } from 'common/utils';
 import { PAITrainingService } from '../pai/paiTrainingService';
 import { RemoteMachineTrainingService } from '../remote_machine/remoteMachineTrainingService';
 import { KubeflowTrainingService } from '../kubernetes/kubeflow/kubeflowTrainingService';

ts/nni_manager/training_service/reusable/sharedStorage.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import { SharedStorageConfig } from '../../common/experimentConfig';
+import { SharedStorageConfig } from 'common/experimentConfig';
 import { StorageService } from './storageService'
 
 export type SharedStorageType = 'NFS' | 'AzureBlob'

ts/nni_manager/training_service/reusable/shared_storages/azureblobStorageService.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import * as cpp from 'child-process-promise';
-import * as path from 'path';
+import cpp from 'child-process-promise';
+import path from 'path';
 
 import { SharedStorageService, SharedStorageType } from '../sharedStorage'
 import { MountedStorageService } from '../storages/mountedStorageService';
-import { getLogger, Logger } from '../../../common/log';
-import { getExperimentId } from '../../../common/experimentStartupInfo';
-import { AzureBlobConfig } from '../../../common/experimentConfig';
+import { getLogger, Logger } from 'common/log';
+import { getExperimentId } from 'common/experimentStartupInfo';
+import { AzureBlobConfig } from 'common/experimentConfig';
 
 const INSTALL_BLOBFUSE = `
 #!/bin/bash

ts/nni_manager/training_service/reusable/shared_storages/nfsStorageService.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import * as cpp from 'child-process-promise';
-import * as path from 'path';
+import cpp from 'child-process-promise';
+import path from 'path';
 
 import { SharedStorageService, SharedStorageType } from '../sharedStorage'
 import { MountedStorageService } from '../storages/mountedStorageService';
-import { getLogger, Logger } from '../../../common/log';
-import { getExperimentId } from '../../../common/experimentStartupInfo';
-import { NfsConfig } from '../../../common/experimentConfig';
+import { getLogger, Logger } from 'common/log';
+import { getExperimentId } from 'common/experimentStartupInfo';
+import { NfsConfig } from 'common/experimentConfig';
 
 const INSTALL_NFS_CLIENT = `
 #!/bin/bash

ts/nni_manager/training_service/reusable/storageService.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import * as fs from 'fs';
-import * as os from 'os';
-import * as path from 'path';
-import { getLogger, Logger } from '../../common/log';
-import { uniqueString } from '../../common/utils';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import { getLogger, Logger } from 'common/log';
+import { uniqueString } from 'common/utils';
 import { tarAdd } from '../common/util';
 
 export abstract class StorageService {

ts/nni_manager/training_service/reusable/storages/mountedStorageService.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-import * as fs from 'fs';
-import * as path from 'path';
+import fs from 'fs';
+import path from 'path';
 import { Deferred } from "ts-deferred";
 import { StorageService } from "../storageService";
 

ts/nni_manager/training_service/reusable/trial.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
-import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from "../../common/trainingService";
-import { GPUInfo } from "../../training_service/common/gpuData";
+import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from "common/trainingService";
+import { GPUInfo } from "training_service/common/gpuData";
 import { EnvironmentInformation, NodeInformation } from "./environment";
 
 export class TrialDetail implements TrialJobDetail {

ts/nni_manager/training_service/reusable/trialDispatcher.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-'use strict';
-
 import { EventEmitter } from 'events';
-import * as fs from 'fs';
-import * as path from 'path';
+import fs from 'fs';
+import path from 'path';
 import { Writable } from 'stream';
 import { Container, Scope } from 'typescript-ioc';
 import { String } from 'typescript-string-operations';
-import * as component from '../../common/component';
-import { NNIError, NNIErrorNames, MethodNotImplementedError } from '../../common/errors';
-import { getBasePort, getExperimentId } from '../../common/experimentStartupInfo';
-import { getLogger, Logger } from '../../common/log';
-import { TrainingService, TrialJobApplicationForm, TrialJobMetric, TrialJobStatus } from '../../common/trainingService';
-import { delay, getExperimentRootDir, getIPV4Address, getLogLevel, getVersion, mkDirPSync, randomSelect, uniqueString } from '../../common/utils';
-import { ExperimentConfig, SharedStorageConfig } from '../../common/experimentConfig';
-import { GPU_INFO, INITIALIZED, KILL_TRIAL_JOB, NEW_TRIAL_JOB, REPORT_METRIC_DATA, SEND_TRIAL_JOB_PARAMETER, STDOUT, TRIAL_END, VERSION_CHECK } from '../../core/commands';
-import { ScheduleResultType } from '../../training_service/common/gpuData';
+import * as component from 'common/component';
+import { NNIError, NNIErrorNames, MethodNotImplementedError } from 'common/errors';
+import { getBasePort, getExperimentId } from 'common/experimentStartupInfo';
+import { getLogger, Logger } from 'common/log';
+import { TrainingService, TrialJobApplicationForm, TrialJobMetric, TrialJobStatus } from 'common/trainingService';
+import { delay, getExperimentRootDir, getIPV4Address, getLogLevel, getVersion, mkDirPSync, randomSelect, uniqueString } from 'common/utils';
+import { ExperimentConfig, SharedStorageConfig } from 'common/experimentConfig';
+import { GPU_INFO, INITIALIZED, KILL_TRIAL_JOB, NEW_TRIAL_JOB, REPORT_METRIC_DATA, SEND_TRIAL_JOB_PARAMETER, STDOUT, TRIAL_END, VERSION_CHECK } from 'core/commands';
+import { ScheduleResultType } from 'training_service/common/gpuData';
 import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../common/containerJobData';
 import { CONTAINER_INSTALL_NNI_SHELL_FORMAT_FOR_WIN } from '../common/containerJobData';
 import { TrialConfig } from '../common/trialConfig';

ts/nni_manager/tsconfig.json

 {
+  "include": [
+      "common/**/*",
+      "core/**/*",
+      "rest_server/**/*",
+      "training_service/**/*",
+      "main.ts"
+  ],
   "compilerOptions": {
-    "target": "ESNext",
-    "module": "CommonJS",
-    "removeComments": true,
+    "allowUnreachableCode": false,
+    "noImplicitReturns": true,
+    "noPropertyAccessFromIndexSignature": true,
+    "noUnusedParameters": true,
     "strict": true,
-    "noImplicitAny": true,
     "baseUrl": ".",
-    "outDir": "dist",
-    "experimentalDecorators": true,
-    "emitDecoratorMetadata": true,
+    "module": "CommonJS",
     "typeRoots": [
       "node_modules/@types",
       "types"
-    ]
-  },
-  "exclude": [
-    "node_modules",
-    "test"
-  ]
+    ],
+    "outDir": "dist",
+    "removeComments": true,
+    "allowSyntheticDefaultImports": true,
+    "esModuleInterop": true,
+    "emitDecoratorMetadata": true,
+    "experimentalDecorators": true,
+    "target": "ESNext",
+    "incremental": true,
+    "tsBuildInfoFile": "./.tsbuildinfo"
+  }
 }