Unverified Commit 5fc73ba6 authored by liuzhe-lz's avatar liuzhe-lz Committed by GitHub
Browse files

Refactor NNI Manager UT (Stage 1) - setup toolchain (#4109)

parent 2b9f5f8c
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from 'common/trainingService';
import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from '../../common/trainingService';
import {TrialConfig} from '../common/trialConfig'; import {TrialConfig} from '../common/trialConfig';
export class PAIClusterConfig { export class PAIClusterConfig {
......
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import request from 'request';
import * as request from 'request';
import { Deferred } from 'ts-deferred'; import { Deferred } from 'ts-deferred';
import { NNIError, NNIErrorNames } from '../../common/errors'; import { NNIError, NNIErrorNames } from 'common/errors';
import { getLogger, Logger } from '../../common/log'; import { getLogger, Logger } from 'common/log';
import { TrialJobStatus } from '../../common/trainingService'; import { TrialJobStatus } from 'common/trainingService';
import { ExperimentConfig, OpenpaiConfig } from '../../common/experimentConfig'; import { ExperimentConfig, OpenpaiConfig } from 'common/experimentConfig';
import { PAITrialJobDetail } from './paiConfig'; import { PAITrialJobDetail } from './paiConfig';
interface FlattenOpenpaiConfig extends ExperimentConfig, OpenpaiConfig { } interface FlattenOpenpaiConfig extends ExperimentConfig, OpenpaiConfig { }
...@@ -44,7 +42,7 @@ export class PAIJobInfoCollector { ...@@ -44,7 +42,7 @@ export class PAIJobInfoCollector {
await Promise.all(updatePaiTrialJobs); await Promise.all(updatePaiTrialJobs);
} }
private getSinglePAITrialJobInfo(protocol: string, paiTrialJob: PAITrialJobDetail, paiToken: string, config: FlattenOpenpaiConfig): Promise<void> { private getSinglePAITrialJobInfo(_protocol: string, paiTrialJob: PAITrialJobDetail, paiToken: string, config: FlattenOpenpaiConfig): Promise<void> {
const deferred: Deferred<void> = new Deferred<void>(); const deferred: Deferred<void> = new Deferred<void>();
if (!this.statusesNeedToCheck.includes(paiTrialJob.status)) { if (!this.statusesNeedToCheck.includes(paiTrialJob.status)) {
deferred.resolve(); deferred.resolve();
......
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict';
import { Request, Response, Router } from 'express'; import { Request, Response, Router } from 'express';
import { ClusterJobRestServer } from '../common/clusterJobRestServer'; import { ClusterJobRestServer } from '../common/clusterJobRestServer';
import { PAITrainingService } from './paiTrainingService'; import { PAITrainingService } from './paiTrainingService';
...@@ -56,7 +54,7 @@ export class PAIJobRestServer extends ClusterJobRestServer { ...@@ -56,7 +54,7 @@ export class PAIJobRestServer extends ClusterJobRestServer {
} }
}); });
router.get(`/parameter-file-meta`, (req: Request, res: Response) => { router.get(`/parameter-file-meta`, (_req: Request, res: Response) => {
try { try {
this.log.info(`GET /parameter-file-meta`); this.log.info(`GET /parameter-file-meta`);
res.send(this.parameterFileMetaList); res.send(this.parameterFileMetaList);
......
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import fs from 'fs';
import path from 'path';
import * as fs from 'fs'; import request from 'request';
import * as path from 'path'; import * as component from 'common/component';
import * as request from 'request';
import * as component from '../../common/component';
import { EventEmitter } from 'events'; import { EventEmitter } from 'events';
import { Deferred } from 'ts-deferred'; import { Deferred } from 'ts-deferred';
import { getExperimentId } from '../../common/experimentStartupInfo'; import { getExperimentId } from 'common/experimentStartupInfo';
import { getLogger, Logger } from '../../common/log'; import { getLogger, Logger } from 'common/log';
import { MethodNotImplementedError } from '../../common/errors'; import { MethodNotImplementedError } from 'common/errors';
import { import {
HyperParameters, NNIManagerIpConfig, TrainingService, HyperParameters, NNIManagerIpConfig, TrainingService,
TrialJobApplicationForm, TrialJobDetail, TrialJobMetric TrialJobApplicationForm, TrialJobDetail, TrialJobMetric
} from '../../common/trainingService'; } from 'common/trainingService';
import { delay } from '../../common/utils'; import { delay } from 'common/utils';
import { ExperimentConfig, OpenpaiConfig, flattenConfig, toMegaBytes } from '../../common/experimentConfig'; import { ExperimentConfig, OpenpaiConfig, flattenConfig, toMegaBytes } from 'common/experimentConfig';
import { PAIJobInfoCollector } from './paiJobInfoCollector'; import { PAIJobInfoCollector } from './paiJobInfoCollector';
import { PAIJobRestServer } from './paiJobRestServer'; import { PAIJobRestServer } from './paiJobRestServer';
import { PAITrialJobDetail, PAI_TRIAL_COMMAND_FORMAT } from './paiConfig'; import { PAITrialJobDetail, PAI_TRIAL_COMMAND_FORMAT } from './paiConfig';
import { String } from 'typescript-string-operations'; import { String } from 'typescript-string-operations';
import { generateParamFileName, getIPV4Address, uniqueString } from '../../common/utils'; import { generateParamFileName, getIPV4Address, uniqueString } from 'common/utils';
import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../common/containerJobData'; import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../common/containerJobData';
import { execMkdir, validateCodeDir, execCopydir } from '../common/util'; import { execMkdir, validateCodeDir, execCopydir } from '../common/util';
......
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict';
import { OsCommands } from "../osCommands"; import { OsCommands } from "../osCommands";
import { RemoteCommandResult } from "../remoteMachineData"; import { RemoteCommandResult } from "../remoteMachineData";
......
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict';
import { OsCommands } from "../osCommands"; import { OsCommands } from "../osCommands";
import { RemoteCommandResult } from "../remoteMachineData"; import { RemoteCommandResult } from "../remoteMachineData";
......
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import assert from 'assert';
import { getLogger, Logger } from 'common/log';
import * as assert from 'assert'; import { randomSelect } from 'common/utils';
import { getLogger, Logger } from '../../common/log'; import { RemoteMachineConfig } from 'common/experimentConfig';
import { randomSelect } from '../../common/utils';
import { RemoteMachineConfig } from '../../common/experimentConfig';
import { GPUInfo, ScheduleResultType } from '../common/gpuData'; import { GPUInfo, ScheduleResultType } from '../common/gpuData';
import { ExecutorManager, RemoteMachineMeta, RemoteMachineScheduleResult, RemoteMachineTrialJobDetail } from './remoteMachineData'; import { ExecutorManager, RemoteMachineMeta, RemoteMachineScheduleResult, RemoteMachineTrialJobDetail } from './remoteMachineData';
...@@ -123,6 +121,7 @@ export class GPUScheduler { ...@@ -123,6 +121,7 @@ export class GPUScheduler {
assert(false, 'gpuInfos is undefined'); assert(false, 'gpuInfos is undefined');
} }
} }
return undefined;
} }
/** /**
......
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict';
import { RemoteCommandResult } from "./remoteMachineData"; import { RemoteCommandResult } from "./remoteMachineData";
abstract class OsCommands { abstract class OsCommands {
......
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from 'common/trainingService';
import { RemoteMachineConfig } from 'common/experimentConfig';
import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from '../../common/trainingService';
import { RemoteMachineConfig } from '../../common/experimentConfig';
import { GPUInfo, GPUSummary, ScheduleResultType } from '../common/gpuData'; import { GPUInfo, GPUSummary, ScheduleResultType } from '../common/gpuData';
import { ShellExecutor } from './shellExecutor'; import { ShellExecutor } from './shellExecutor';
......
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict';
import { ClusterJobRestServer } from '../common/clusterJobRestServer'; import { ClusterJobRestServer } from '../common/clusterJobRestServer';
import { RemoteMachineTrainingService } from './remoteMachineTrainingService'; import { RemoteMachineTrainingService } from './remoteMachineTrainingService';
......
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import assert from 'assert';
import * as assert from 'assert';
import { EventEmitter } from 'events'; import { EventEmitter } from 'events';
import * as fs from 'fs'; import fs from 'fs';
import * as path from 'path'; import path from 'path';
import { ShellExecutor } from 'training_service/remote_machine/shellExecutor'; import { ShellExecutor } from 'training_service/remote_machine/shellExecutor';
import { Deferred } from 'ts-deferred'; import { Deferred } from 'ts-deferred';
import * as component from '../../common/component'; import * as component from 'common/component';
import { NNIError, NNIErrorNames, MethodNotImplementedError } from '../../common/errors'; import { NNIError, NNIErrorNames, MethodNotImplementedError } from 'common/errors';
import { getExperimentId } from '../../common/experimentStartupInfo'; import { getExperimentId } from 'common/experimentStartupInfo';
import { getLogger, Logger } from '../../common/log'; import { getLogger, Logger } from 'common/log';
import { ObservableTimer } from '../../common/observableTimer'; import { ObservableTimer } from 'common/observableTimer';
import { import {
HyperParameters, TrainingService, TrialJobApplicationForm, HyperParameters, TrainingService, TrialJobApplicationForm,
TrialJobDetail, TrialJobMetric TrialJobDetail, TrialJobMetric
} from '../../common/trainingService'; } from 'common/trainingService';
import { import {
delay, generateParamFileName, getExperimentRootDir, getIPV4Address, getJobCancelStatus, delay, generateParamFileName, getExperimentRootDir, getIPV4Address, getJobCancelStatus,
getVersion, uniqueString getVersion, uniqueString
} from '../../common/utils'; } from 'common/utils';
import { ExperimentConfig, RemoteConfig, RemoteMachineConfig, flattenConfig } from '../../common/experimentConfig'; import { ExperimentConfig, RemoteConfig, RemoteMachineConfig, flattenConfig } from 'common/experimentConfig';
import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../common/containerJobData'; import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../common/containerJobData';
import { GPUSummary, ScheduleResultType } from '../common/gpuData'; import { GPUSummary, ScheduleResultType } from '../common/gpuData';
import { execMkdir, validateCodeDir } from '../common/util'; import { execMkdir, validateCodeDir } from '../common/util';
......
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import assert from 'assert';
import fs from 'fs';
import * as assert from 'assert'; import os from 'os';
import * as fs from 'fs'; import path from 'path';
import * as os from 'os';
import * as path from 'path';
import { Client, ClientChannel, ConnectConfig, SFTPWrapper } from 'ssh2'; import { Client, ClientChannel, ConnectConfig, SFTPWrapper } from 'ssh2';
import * as stream from 'stream'; import stream from 'stream';
import { Deferred } from "ts-deferred"; import { Deferred } from "ts-deferred";
import { getLogger, Logger } from '../../common/log'; import { getLogger, Logger } from 'common/log';
import { uniqueString, randomInt } from '../../common/utils'; import { uniqueString, randomInt } from 'common/utils';
import { execRemove, tarAdd } from '../common/util'; import { execRemove, tarAdd } from '../common/util';
import { LinuxCommands } from "./extends/linuxCommands"; import { LinuxCommands } from "./extends/linuxCommands";
import { WindowsCommands } from './extends/windowsCommands'; import { WindowsCommands } from './extends/windowsCommands';
import { OsCommands } from "./osCommands"; import { OsCommands } from "./osCommands";
import { RemoteCommandResult, RemoteMachineMeta } from "./remoteMachineData"; import { RemoteCommandResult, RemoteMachineMeta } from "./remoteMachineData";
import { NNIError, NNIErrorNames } from '../../common/errors'; import { NNIError, NNIErrorNames } from 'common/errors';
class ShellExecutor { class ShellExecutor {
public name: string = ""; public name: string = "";
......
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict';
import { Deferred } from 'ts-deferred'; import { Deferred } from 'ts-deferred';
import { PythonShell } from 'python-shell'; import { PythonShell } from 'python-shell';
......
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import { TrialConfig } from 'training_service/common/trialConfig';
import { TrialConfig } from '../../common/trialConfig';
import { EnvironmentInformation } from '../environment'; import { EnvironmentInformation } from '../environment';
import { AMLClient } from '../aml/amlClient'; import { AMLClient } from '../aml/amlClient';
......
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import { delay } from "common/utils";
import { delay } from "../../../common/utils";
import { AMLEnvironmentInformation } from '../aml/amlConfig'; import { AMLEnvironmentInformation } from '../aml/amlConfig';
import { CommandChannel, RunnerConnection } from "../commandChannel"; import { CommandChannel, RunnerConnection } from "../commandChannel";
import { Channel, EnvironmentInformation } from "../environment"; import { Channel, EnvironmentInformation } from "../environment";
......
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import * as component from "common/component";
import { delay } from "common/utils";
import * as component from "../../../common/component";
import { delay } from "../../../common/utils";
import { CommandChannel, RunnerConnection } from "../commandChannel"; import { CommandChannel, RunnerConnection } from "../commandChannel";
import { Channel, EnvironmentInformation } from "../environment"; import { Channel, EnvironmentInformation } from "../environment";
import { StorageService } from "../storageService"; import { StorageService } from "../storageService";
......
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict';
import { Server as SocketServer } from "ws"; import { Server as SocketServer } from "ws";
import { getBasePort, getExperimentId } from "../../../common/experimentStartupInfo"; import { getBasePort, getExperimentId } from "common/experimentStartupInfo";
import { INITIALIZED } from '../../../core/commands'; import { INITIALIZED } from 'core/commands';
import { CommandChannel, RunnerConnection } from "../commandChannel"; import { CommandChannel, RunnerConnection } from "../commandChannel";
import { Channel, EnvironmentInformation } from "../environment"; import { Channel, EnvironmentInformation } from "../environment";
import { EventEmitter } from "events"; import { EventEmitter } from "events";
......
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict';
import { EventEmitter } from "events"; import { EventEmitter } from "events";
import { getLogger, Logger } from "../../common/log"; import { getLogger, Logger } from "common/log";
import { TRIAL_COMMANDS } from "../../core/commands"; import { TRIAL_COMMANDS } from "core/commands";
import { encodeCommand } from "../../core/ipcInterface"; import { encodeCommand } from "core/ipcInterface";
import { Channel, EnvironmentInformation } from "./environment"; import { Channel, EnvironmentInformation } from "./environment";
const acceptedCommands: Set<string> = new Set<string>(TRIAL_COMMANDS); const acceptedCommands: Set<string> = new Set<string>(TRIAL_COMMANDS);
......
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict';
import { Deferred } from 'ts-deferred'; import { Deferred } from 'ts-deferred';
import { PythonShell } from 'python-shell'; import { PythonShell } from 'python-shell';
import { getLogger, Logger } from '../../../common/log'; import { getLogger, Logger } from 'common/log';
export class DlcClient { export class DlcClient {
private log: Logger; private log: Logger;
......
// Copyright (c) Microsoft Corporation. // Copyright (c) Microsoft Corporation.
// Licensed under the MIT license. // Licensed under the MIT license.
'use strict'; import { TrialConfig } from 'training_service/common/trialConfig';
import { TrialConfig } from '../../common/trialConfig';
import { EnvironmentInformation } from '../environment'; import { EnvironmentInformation } from '../environment';
import { DlcClient } from '../dlc/dlcClient'; import { DlcClient } from '../dlc/dlcClient';
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment