Skip to content

GitLab

Unverified Commit 5fc73ba6 authored by liuzhe-lz's avatar liuzhe-lz Committed by GitHub
Browse files

Refactor NNI Manager UT (Stage 1) - setup toolchain (#4109)

parent 2b9f5f8c
Hide whitespace changes
Inline Side-by-side
Showing with 76 additions and 115 deletions
ts/nni_manager/training_service/kubernetes/adl/adlTrainingService.ts
View file @ 5fc73ba6
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import fs from 'fs';
import * as component from 'common/component';
import * as fs from 'fs';
import * as component from '../../../common/component';
import { String } from 'typescript-string-operations'; import { String } from 'typescript-string-operations';
import { getExperimentId } from '../../../common/experimentStartupInfo'; import { getExperimentId } from 'common/experimentStartupInfo';
import { import {
NNIManagerIpConfig, TrialJobApplicationForm, TrialJobDetail, TrialJobStatus NNIManagerIpConfig, TrialJobApplicationForm, TrialJobDetail, TrialJobStatus
} from '../../../common/trainingService'; } from 'common/trainingService';
import { delay, generateParamFileName, getVersion, uniqueString } from '../../../common/utils'; import { delay, generateParamFileName, getVersion, uniqueString } from 'common/utils';
import { TrialConfigMetadataKey } from '../../common/trialConfigMetadataKey'; import { TrialConfigMetadataKey } from 'training_service/common/trialConfigMetadataKey';
import { KubernetesTrialJobDetail } from '../kubernetesData'; import { KubernetesTrialJobDetail } from '../kubernetesData';
import { KubernetesTrainingService } from '../kubernetesTrainingService'; import { KubernetesTrainingService } from '../kubernetesTrainingService';
import { AdlClientFactory } from './adlApiClient' import { AdlClientFactory } from './adlApiClient'
......
ts/nni_manager/training_service/kubernetes/azureStorageClientUtils.ts
View file @ 5fc73ba6
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import azureStorage from 'azure-storage';
import fs from 'fs';
import * as azureStorage from 'azure-storage'; import path from 'path';
import * as fs from 'fs';
import * as path from 'path';
import { Deferred } from 'ts-deferred'; import { Deferred } from 'ts-deferred';
import { String } from 'typescript-string-operations'; import { String } from 'typescript-string-operations';
import { getLogger } from '../../common/log'; import { getLogger } from 'common/log';
import { mkDirP } from '../../common/utils'; import { mkDirP } from 'common/utils';
export namespace AzureStorageClientUtility { export namespace AzureStorageClientUtility {
......
ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerApiClient.ts
View file @ 5fc73ba6
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import fs from 'fs';
import * as fs from 'fs';
import {GeneralK8sClient, KubernetesCRDClient} from '../kubernetesApiClient'; import {GeneralK8sClient, KubernetesCRDClient} from '../kubernetesApiClient';
/** /**
......
ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerConfig.ts
View file @ 5fc73ba6
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import assert from 'assert';
import * as assert from 'assert';
import { import {
AzureStorage, KeyVaultConfig, KubernetesClusterConfig, KubernetesClusterConfigAzure, KubernetesClusterConfigNFS, AzureStorage, KeyVaultConfig, KubernetesClusterConfig, KubernetesClusterConfigAzure, KubernetesClusterConfigNFS,
...@@ -49,7 +47,7 @@ export class FrameworkControllerTrialConfig extends KubernetesTrialConfig { ...@@ -49,7 +47,7 @@ export class FrameworkControllerTrialConfig extends KubernetesTrialConfig {
export class FrameworkControllerClusterConfig extends KubernetesClusterConfig { export class FrameworkControllerClusterConfig extends KubernetesClusterConfig {
public readonly serviceAccountName: string; public readonly serviceAccountName: string;
constructor(apiVersion: string, serviceAccountName: string, configPath?: string, namespace?: string) { constructor(apiVersion: string, serviceAccountName: string, _configPath?: string, namespace?: string) {
super(apiVersion, undefined, namespace); super(apiVersion, undefined, namespace);
this.serviceAccountName = serviceAccountName; this.serviceAccountName = serviceAccountName;
} }
......
ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerJobInfoCollector.ts
View file @ 5fc73ba6
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict';
import { KubernetesCRDClient } from '../kubernetesApiClient'; import { KubernetesCRDClient } from '../kubernetesApiClient';
import { KubernetesTrialJobDetail} from '../kubernetesData'; import { KubernetesTrialJobDetail} from '../kubernetesData';
import { KubernetesJobInfoCollector } from '../kubernetesJobInfoCollector'; import { KubernetesJobInfoCollector } from '../kubernetesJobInfoCollector';
......
ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerJobRestServer.ts
View file @ 5fc73ba6
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import * as component from 'common/component';
import * as component from '../../../common/component';
import { KubernetesJobRestServer } from '../kubernetesJobRestServer'; import { KubernetesJobRestServer } from '../kubernetesJobRestServer';
import { FrameworkControllerTrainingService } from './frameworkcontrollerTrainingService'; import { FrameworkControllerTrainingService } from './frameworkcontrollerTrainingService';
......
ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts
View file @ 5fc73ba6
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import assert from 'assert';
import cpp from 'child-process-promise';
import * as assert from 'assert'; import fs from 'fs';
import * as cpp from 'child-process-promise'; import path from 'path';
import * as fs from 'fs'; import * as component from 'common/component';
import * as path from 'path'; import {getExperimentId} from 'common/experimentStartupInfo';
import * as component from '../../../common/component';
import {getExperimentId} from '../../../common/experimentStartupInfo';
import { import {
NNIManagerIpConfig, TrialJobApplicationForm, TrialJobDetail, TrialJobStatus NNIManagerIpConfig, TrialJobApplicationForm, TrialJobDetail, TrialJobStatus
} from '../../../common/trainingService'; } from 'common/trainingService';
import {delay, generateParamFileName, getExperimentRootDir, uniqueString} from '../../../common/utils'; import {delay, generateParamFileName, getExperimentRootDir, uniqueString} from 'common/utils';
import {CONTAINER_INSTALL_NNI_SHELL_FORMAT} from '../../common/containerJobData'; import {CONTAINER_INSTALL_NNI_SHELL_FORMAT} from 'training_service/common/containerJobData';
import {TrialConfigMetadataKey} from '../../common/trialConfigMetadataKey'; import {TrialConfigMetadataKey} from 'training_service/common/trialConfigMetadataKey';
import {validateCodeDir} from '../../common/util'; import {validateCodeDir} from 'training_service/common/util';
import {NFSConfig} from '../kubernetesConfig'; import {NFSConfig} from '../kubernetesConfig';
import {KubernetesTrialJobDetail} from '../kubernetesData'; import {KubernetesTrialJobDetail} from '../kubernetesData';
import {KubernetesTrainingService} from '../kubernetesTrainingService'; import {KubernetesTrainingService} from '../kubernetesTrainingService';
......
ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowApiClient.ts
View file @ 5fc73ba6
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import fs from 'fs';
import * as fs from 'fs';
import { GeneralK8sClient, KubernetesCRDClient } from '../kubernetesApiClient'; import { GeneralK8sClient, KubernetesCRDClient } from '../kubernetesApiClient';
import { KubeflowOperator } from './kubeflowConfig'; import { KubeflowOperator } from './kubeflowConfig';
......
ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowConfig.ts
View file @ 5fc73ba6
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import assert from 'assert';
import { MethodNotImplementedError } from 'common/errors';
import * as assert from 'assert';
import { MethodNotImplementedError } from '../../../common/errors';
import { AzureStorage, KeyVaultConfig, KubernetesClusterConfig, KubernetesClusterConfigAzure, KubernetesClusterConfigNFS, import { AzureStorage, KeyVaultConfig, KubernetesClusterConfig, KubernetesClusterConfigAzure, KubernetesClusterConfigNFS,
KubernetesStorageKind, KubernetesTrialConfig, KubernetesTrialConfigTemplate, NFSConfig, StorageConfig KubernetesStorageKind, KubernetesTrialConfig, KubernetesTrialConfigTemplate, NFSConfig, StorageConfig
} from '../kubernetesConfig'; } from '../kubernetesConfig';
......
ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowJobInfoCollector.ts
View file @ 5fc73ba6
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict';
import { KubernetesCRDClient } from '../kubernetesApiClient'; import { KubernetesCRDClient } from '../kubernetesApiClient';
import { KubernetesTrialJobDetail} from '../kubernetesData'; import { KubernetesTrialJobDetail} from '../kubernetesData';
import { KubernetesJobInfoCollector } from '../kubernetesJobInfoCollector'; import { KubernetesJobInfoCollector } from '../kubernetesJobInfoCollector';
......
ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowJobRestServer.ts
View file @ 5fc73ba6
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import * as component from 'common/component';
import * as component from '../../../common/component';
import { KubernetesJobRestServer } from '../kubernetesJobRestServer'; import { KubernetesJobRestServer } from '../kubernetesJobRestServer';
import { KubeflowTrainingService } from './kubeflowTrainingService'; import { KubeflowTrainingService } from './kubeflowTrainingService';
......
ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts
View file @ 5fc73ba6
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import assert from 'assert';
import cpp from 'child-process-promise';
import fs from 'fs';
import path from 'path';
import * as component from 'common/component';
import * as assert from 'assert'; import { getExperimentId } from 'common/experimentStartupInfo';
import * as cpp from 'child-process-promise';
import * as fs from 'fs';
import * as path from 'path';
import * as component from '../../../common/component';
import { getExperimentId } from '../../../common/experimentStartupInfo';
import { import {
NNIManagerIpConfig, TrialJobApplicationForm, TrialJobDetail, TrialJobStatus NNIManagerIpConfig, TrialJobApplicationForm, TrialJobDetail, TrialJobStatus
} from '../../../common/trainingService'; } from 'common/trainingService';
import { delay, generateParamFileName, getExperimentRootDir, uniqueString } from '../../../common/utils'; import { delay, generateParamFileName, getExperimentRootDir, uniqueString } from 'common/utils';
import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../../common/containerJobData'; import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from 'training_service/common/containerJobData';
import { TrialConfigMetadataKey } from '../../common/trialConfigMetadataKey'; import { TrialConfigMetadataKey } from 'training_service/common/trialConfigMetadataKey';
import { validateCodeDir } from '../../common/util'; import { validateCodeDir } from 'training_service/common/util';
import { NFSConfig } from '../kubernetesConfig'; import { NFSConfig } from '../kubernetesConfig';
import { KubernetesTrialJobDetail } from '../kubernetesData'; import { KubernetesTrialJobDetail } from '../kubernetesData';
import { KubernetesJobRestServer } from '../kubernetesJobRestServer'; import { KubernetesJobRestServer } from '../kubernetesJobRestServer';
......
ts/nni_manager/training_service/kubernetes/kubernetesApiClient.ts
View file @ 5fc73ba6
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict';
// eslint-disable-next-line @typescript-eslint/camelcase // eslint-disable-next-line @typescript-eslint/camelcase
import {Client1_10, config} from 'kubernetes-client'; import {Client1_10, config} from 'kubernetes-client';
import {getLogger, Logger} from '../../common/log'; import {getLogger, Logger} from 'common/log';
/** /**
* This function uses the environment variable * This function uses the environment variable
......
ts/nni_manager/training_service/kubernetes/kubernetesConfig.ts
View file @ 5fc73ba6
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict';
export type KubernetesStorageKind = 'nfs' | 'azureStorage' | 'pvc'; export type KubernetesStorageKind = 'nfs' | 'azureStorage' | 'pvc';
import {MethodNotImplementedError} from '../../common/errors'; import {MethodNotImplementedError} from 'common/errors';
export abstract class KubernetesClusterConfig { export abstract class KubernetesClusterConfig {
public readonly storage?: KubernetesStorageKind; public readonly storage?: KubernetesStorageKind;
......
ts/nni_manager/training_service/kubernetes/kubernetesData.ts
View file @ 5fc73ba6
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from 'common/trainingService';
import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from '../../common/trainingService';
/** /**
* KubeflowTrialJobDetail * KubeflowTrialJobDetail
......
ts/nni_manager/training_service/kubernetes/kubernetesJobInfoCollector.ts
View file @ 5fc73ba6
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import assert from 'assert';
import { MethodNotImplementedError, NNIError, NNIErrorNames } from 'common/errors';
import * as assert from 'assert'; import { getLogger, Logger } from 'common/log';
import { MethodNotImplementedError, NNIError, NNIErrorNames } from '../../common/errors'; import { TrialJobStatus } from 'common/trainingService';
import { getLogger, Logger } from '../../common/log';
import { TrialJobStatus } from '../../common/trainingService';
import { KubernetesCRDClient } from './kubernetesApiClient'; import { KubernetesCRDClient } from './kubernetesApiClient';
import { KubernetesTrialJobDetail } from './kubernetesData'; import { KubernetesTrialJobDetail } from './kubernetesData';
......
ts/nni_manager/training_service/kubernetes/kubernetesJobRestServer.ts
View file @ 5fc73ba6
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict';
import { Inject } from 'typescript-ioc'; import { Inject } from 'typescript-ioc';
import * as component from '../../common/component'; import * as component from 'common/component';
import { ClusterJobRestServer } from '../common/clusterJobRestServer'; import { ClusterJobRestServer } from '../common/clusterJobRestServer';
import { KubernetesTrainingService } from './kubernetesTrainingService'; import { KubernetesTrainingService } from './kubernetesTrainingService';
......
ts/nni_manager/training_service/kubernetes/kubernetesTrainingService.ts
View file @ 5fc73ba6
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import cpp from 'child-process-promise';
import path from 'path';
import * as cpp from 'child-process-promise'; import azureStorage from 'azure-storage';
import * as path from 'path';
import * as azureStorage from 'azure-storage';
import {EventEmitter} from 'events'; import {EventEmitter} from 'events';
import {Base64} from 'js-base64'; import {Base64} from 'js-base64';
import {String} from 'typescript-string-operations'; import {String} from 'typescript-string-operations';
import {getExperimentId} from '../../common/experimentStartupInfo'; import {getExperimentId} from 'common/experimentStartupInfo';
import {getLogger, Logger} from '../../common/log'; import {getLogger, Logger} from 'common/log';
import {MethodNotImplementedError} from '../../common/errors'; import {MethodNotImplementedError} from 'common/errors';
import { import {
NNIManagerIpConfig, TrialJobDetail, TrialJobMetric NNIManagerIpConfig, TrialJobDetail, TrialJobMetric
} from '../../common/trainingService'; } from 'common/trainingService';
import {delay, getExperimentRootDir, getIPV4Address, getJobCancelStatus, getVersion, uniqueString} from '../../common/utils'; import {delay, getExperimentRootDir, getIPV4Address, getJobCancelStatus, getVersion, uniqueString} from 'common/utils';
import {AzureStorageClientUtility} from './azureStorageClientUtils'; import {AzureStorageClientUtility} from './azureStorageClientUtils';
import {GeneralK8sClient, KubernetesCRDClient} from './kubernetesApiClient'; import {GeneralK8sClient, KubernetesCRDClient} from './kubernetesApiClient';
import {KubernetesClusterConfig} from './kubernetesConfig'; import {KubernetesClusterConfig} from './kubernetesConfig';
......
ts/nni_manager/training_service/local/gpuScheduler.ts
View file @ 5fc73ba6
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import cpp from 'child-process-promise';
import fs from 'fs';
import * as cpp from 'child-process-promise'; import os from 'os';
import * as fs from 'fs'; import path from 'path';
import * as os from 'os'; import { getLogger, Logger } from 'common/log';
import * as path from 'path'; import { delay } from 'common/utils';
import { getLogger, Logger } from '../../common/log';
import { delay } from '../../common/utils';
import { GPUInfo, GPUSummary } from '../common/gpuData'; import { GPUInfo, GPUSummary } from '../common/gpuData';
import { execKill, execMkdir, execRemove, execTail, runGpuMetricsCollector } from '../common/util'; import { execKill, execMkdir, execRemove, execTail, runGpuMetricsCollector } from '../common/util';
......
ts/nni_manager/training_service/local/localTrainingService.ts
View file @ 5fc73ba6
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import cp from 'child_process';
import * as cp from 'child_process';
import { EventEmitter } from 'events'; import { EventEmitter } from 'events';
import * as fs from 'fs'; import fs from 'fs';
import * as path from 'path'; import path from 'path';
import * as ts from 'tail-stream'; import ts from 'tail-stream';
import * as tkill from 'tree-kill'; import tkill from 'tree-kill';
import { NNIError, NNIErrorNames } from '../../common/errors'; import { NNIError, NNIErrorNames } from 'common/errors';
import { getExperimentId } from '../../common/experimentStartupInfo'; import { getExperimentId } from 'common/experimentStartupInfo';
import { getLogger, Logger } from '../../common/log'; import { getLogger, Logger } from 'common/log';
import { powershellString } from '../../common/shellUtils'; import { powershellString } from 'common/shellUtils';
import { import {
HyperParameters, TrainingService, TrialJobApplicationForm, HyperParameters, TrainingService, TrialJobApplicationForm,
TrialJobDetail, TrialJobMetric, TrialJobStatus TrialJobDetail, TrialJobMetric, TrialJobStatus
} from '../../common/trainingService'; } from 'common/trainingService';
import { import {
delay, generateParamFileName, getExperimentRootDir, getJobCancelStatus, getNewLine, isAlive, uniqueString delay, generateParamFileName, getExperimentRootDir, getJobCancelStatus, getNewLine, isAlive, uniqueString
} from '../../common/utils'; } from 'common/utils';
import { ExperimentConfig, LocalConfig, flattenConfig } from '../../common/experimentConfig'; import { ExperimentConfig, LocalConfig, flattenConfig } from 'common/experimentConfig';
import { execMkdir, execNewFile, getScriptName, runScript, setEnvironmentVariable } from '../common/util'; import { execMkdir, execNewFile, getScriptName, runScript, setEnvironmentVariable } from '../common/util';
import { GPUScheduler } from './gpuScheduler'; import { GPUScheduler } from './gpuScheduler';
...@@ -447,7 +446,7 @@ class LocalTrainingService implements TrainingService { ...@@ -447,7 +446,7 @@ class LocalTrainingService implements TrainingService {
if (process.platform !== 'win32') { if (process.platform !== 'win32') {
runScriptContent.push('#!/bin/bash'); runScriptContent.push('#!/bin/bash');
} else { } else {
runScriptContent.push(`$env:PATH=${powershellString(process.env.path!)}`) runScriptContent.push(`$env:PATH=${powershellString(process.env['path']!)}`)
} }
for (const variable of variables) { for (const variable of variables) {
runScriptContent.push(setEnvironmentVariable(variable)); runScriptContent.push(setEnvironmentVariable(variable));
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment