"profiler/vscode:/vscode.git/clone" did not exist on "e6ee659488f8a6368273d8f664057ec5b8e724f5"
Unverified Commit 5fc73ba6 authored by liuzhe-lz's avatar liuzhe-lz Committed by GitHub
Browse files

Refactor NNI Manager UT (Stage 1) - setup toolchain (#4109)

parent 2b9f5f8c
......@@ -165,6 +165,7 @@ stages:
- script: |
python setup.py develop
mkdir -p coverage
displayName: Install NNI
- script: |
......@@ -185,12 +186,14 @@ stages:
python -m pytest ut/sdk/test_compressor_tf.py --cov-config=.coveragerc --cov-append
python -m pytest ut/sdk/test_compressor_torch.py --cov-config=.coveragerc --cov-append
python -m pytest ut/sdk/test_model_speedup.py --cov-config=.coveragerc --cov-append
cp coverage.xml ../coverage/python.xml
displayName: Python unit test
- script: |
set -e
cd ts/nni_manager
yarn test
cp coverage/cobertura-coverage.xml ../../coverage/typescript.xml
displayName: TypeScript unit test
- task: PublishTestResults@2
......@@ -203,7 +206,7 @@ stages:
- task: PublishCodeCoverageResults@1
inputs:
codeCoverageTool: Cobertura
summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/*coverage.xml'
summaryFileLocation: coverage/*
displayName: Publish code coverage results
- script: |
......
# Build result
dist/
/dist
/.tsbuildinfo
# node modules
node_modules/
/node_modules
# test files
.experiment.test
{
"color": true,
"require": "test/register.js",
"timeout": "15s"
}
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as ioc from 'typescript-ioc';
const Inject: (...args: any[]) => any = ioc.Inject;
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { ExperimentProfile, TrialJobStatistics } from './manager';
import { TrialJobDetail, TrialJobStatus } from './trainingService';
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
export namespace NNIErrorNames {
export const NOT_FOUND: string = 'NOT_FOUND';
export const INVALID_JOB_DETAIL: string = 'NO_VALID_JOB_DETAIL_FOUND';
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import assert from 'assert';
import { KubeflowOperator, OperatorApiVersion } from '../training_service/kubernetes/kubeflow/kubeflowConfig'
import { KubernetesStorageKind } from '../training_service/kubernetes/kubernetesConfig';
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
abstract class ExperimentManager {
public abstract getExperimentsInfo(): Promise<JSON>;
public abstract setExperimentPath(newPath: string): void;
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import * as os from 'os';
import * as path from 'path';
import assert from 'assert';
import os from 'os';
import path from 'path';
const API_ROOT_URL: string = '/api/v1/nni';
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as fs from 'fs';
import fs from 'fs';
import { Writable } from 'stream';
import * as util from 'util';
import util from 'util';
/* log level constants */
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { MetricDataRecord, MetricType, TrialJobInfo } from './datastore';
import { TrialJobStatus } from './trainingService';
import { ExperimentConfig } from './experimentConfig';
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as fs from 'fs';
import * as path from 'path';
import fs from 'fs';
import path from 'path';
import { promisify } from 'util';
import { runPythonScript } from './pythonScript';
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as rx from 'rx';
import rx from 'rx';
import * as component from '../common/component';
@component.Singleton
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { spawn } from 'child_process';
import { Logger, getLogger } from './log';
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import * as express from 'express';
import * as http from 'http';
import assert from 'assert';
import express from 'express';
import http from 'http';
import { Deferred } from 'ts-deferred';
import { getLogger, Logger } from './log';
import { getBasePort } from './experimentStartupInfo';
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
// for readability
const singleQuote = "'";
const doubleQuote = '"';
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
interface TensorboardParams {
trials: string;
}
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
/**
* define TrialJobStatus
*/
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import assert from 'assert';
import { randomBytes } from 'crypto';
import * as cpp from 'child-process-promise';
import * as cp from 'child_process';
import cpp from 'child-process-promise';
import cp from 'child_process';
import { ChildProcess, spawn, StdioOptions } from 'child_process';
import * as dgram from 'dgram';
import * as fs from 'fs';
import * as net from 'net';
import * as os from 'os';
import * as path from 'path';
import dgram from 'dgram';
import fs from 'fs';
import net from 'net';
import os from 'os';
import path from 'path';
import * as timersPromises from 'timers/promises';
import * as lockfile from 'lockfile';
import lockfile from 'lockfile';
import { Deferred } from 'ts-deferred';
import { Container } from 'typescript-ioc';
import * as glob from 'glob';
import glob from 'glob';
import { Database, DataStore } from './datastore';
import { getExperimentStartupInfo, setExperimentStartupInfo } from './experimentStartupInfo';
......@@ -243,7 +241,7 @@ function countFilesRecursively(directory: string): Promise<number> {
const deferred: Deferred<number> = new Deferred<number>();
let timeoutId: NodeJS.Timer
const delayTimeout: Promise<number> = new Promise((resolve: Function, reject: Function): void => {
const delayTimeout: Promise<number> = new Promise((_resolve: Function, reject: Function): void => {
// Set timeout and reject the promise once reach timeout (5 seconds)
timeoutId = setTimeout(() => {
reject(new Error(`Timeout: path ${directory} has too many files`));
......@@ -375,11 +373,11 @@ function unixPathJoin(...paths: any[]): string {
*/
function withLockSync(func: Function, filePath: string, lockOpts: {[key: string]: any}, ...args: any): any {
const lockName = path.join(path.dirname(filePath), path.basename(filePath) + `.lock.${process.pid}`);
if (typeof lockOpts.stale === 'number'){
if (typeof lockOpts['stale'] === 'number'){
const lockPath = path.join(path.dirname(filePath), path.basename(filePath) + '.lock.*');
const lockFileNames: string[] = glob.sync(lockPath);
const canLock: boolean = lockFileNames.map((fileName) => {
return fs.existsSync(fileName) && Date.now() - fs.statSync(fileName).mtimeMs < lockOpts.stale;
return fs.existsSync(fileName) && Date.now() - fs.statSync(fileName).mtimeMs < lockOpts['stale'];
}).filter(unexpired=>unexpired === true).length === 0;
if (!canLock) {
throw new Error('File has been locked.');
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import assert from 'assert';
import { ChildProcess } from 'child_process';
import { EventEmitter } from 'events';
import * as net from 'net';
import net from 'net';
import { Readable, Writable } from 'stream';
import { NNIError } from '../common/errors';
import { getLogger, Logger } from '../common/log';
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment