Merge pull request #145 from Microsoft/master

merge master

docs/en_US/PAIMode.md

@@ -43,7 +43,7 @@ paiConfig:
 
 Note: You should set `trainingServicePlatform: pai` in NNI config YAML file if you want to start experiment in pai mode.
 
-Compared with LocalMode and [RemoteMachineMode](RemoteMachineMode.md), trial configuration in pai mode have five additional keys:
+Compared with LocalMode and [RemoteMachineMode](RemoteMachineMode.md), trial configuration in pai mode have these additional keys:
 * cpuNum
     * Required key. Should be positive number based on your trial program's CPU  requirement
 * memoryMB
@@ -55,6 +55,10 @@ Compared with LocalMode and [RemoteMachineMode](RemoteMachineMode.md), trial con
     * Optional key. It specifies the HDFS data direcotry for trial to download data. The format should be something like hdfs://{your HDFS host}:9000/{your data directory}
 * outputDir 
     * Optional key. It specifies the HDFS output directory for trial. Once the trial is completed (either succeed or fail), trial's stdout, stderr will be copied to this directory by NNI sdk automatically. The format should be something like hdfs://{your HDFS host}:9000/{your output directory}
+* virturlCluster
+    * Optional key. Set the virtualCluster of PAI. If omitted, the job will run on default virtual cluster.
+* shmMB
+    * Optional key. Set the shmMB configuration of PAI, it set the shared memory for one task in the task role.
 
 Once complete to fill NNI experiment config file and save (for example, save as exp_pai.yml), then run the following command
 ```

src/nni_manager/common/utils.ts

@@ -44,6 +44,11 @@ function getLogDir(): string{
     return path.join(getExperimentRootDir(), 'log');
 }
 
+function getLogLevel(): string{
+    return getExperimentStartupInfo()
+    .getLogLevel();
+}
+
 function getDefaultDatabaseDir(): string {
     return path.join(getExperimentRootDir(), 'db');
 }
@@ -360,4 +365,4 @@ async function getVersion(): Promise<string> {
 
 export {countFilesRecursively, getRemoteTmpDir, generateParamFileName, getMsgDispatcherCommand, getCheckpointDir,
     getLogDir, getExperimentRootDir, getJobCancelStatus, getDefaultDatabaseDir, getIPV4Address, 
-    mkDirP, delay, prepareUnitTest, parseArg, cleanupUnitTest, uniqueString, randomSelect, getVersion };
+    mkDirP, delay, prepareUnitTest, parseArg, cleanupUnitTest, uniqueString, randomSelect, getLogLevel, getVersion };

src/nni_manager/core/nnimanager.ts

@@ -35,7 +35,7 @@ import {
 import {
     TrainingService, TrialJobApplicationForm, TrialJobDetail, TrialJobMetric, TrialJobStatus
 } from '../common/trainingService';
-import { delay, getCheckpointDir, getExperimentRootDir, getLogDir, getMsgDispatcherCommand, mkDirP } from '../common/utils';
+import { delay, getCheckpointDir, getExperimentRootDir, getLogDir, getMsgDispatcherCommand, mkDirP, getLogLevel } from '../common/utils';
 import {
     ADD_CUSTOMIZED_TRIAL_JOB, INITIALIZE, INITIALIZED, KILL_TRIAL_JOB, NEW_TRIAL_JOB, NO_MORE_TRIAL_JOBS, PING,
     REPORT_METRIC_DATA, REQUEST_TRIAL_JOBS, SEND_TRIAL_JOB_PARAMETER, TERMINATE, TRIAL_END, UPDATE_SEARCH_SPACE
@@ -276,7 +276,8 @@ class NNIManager implements Manager {
         let nniEnv = {
             NNI_MODE: mode,
             NNI_CHECKPOINT_DIRECTORY: dataDirectory,
-            NNI_LOG_DIRECTORY: getLogDir()
+            NNI_LOG_DIRECTORY: getLogDir(),
+            NNI_LOG_LEVEL: getLogLevel()
         };
         let newEnv = Object.assign({}, process.env, nniEnv);
         const tunerProc: ChildProcess = spawn(command, [], {

src/nni_manager/rest_server/restValidationSchemas.ts

@@ -42,6 +42,7 @@ export namespace ValidationSchemas {
                 gpuNum: joi.number().min(0),
                 command: joi.string().min(1),
                 virtualCluster: joi.string(),
+                shmMB: joi.number(),
                 worker: joi.object({
                     replicas: joi.number().min(1).required(),
                     image: joi.string().min(1),
@@ -76,6 +77,7 @@ export namespace ValidationSchemas {
                     outputDir: joi.string(),
                     cpuNum: joi.number().min(1),
                     memoryMB: joi.number().min(100),
+                    shmMB: joi.number(),
                     gpuNum: joi.number().min(0).required(),
                     command: joi.string().min(1).required(),
                     frameworkAttemptCompletionPolicy: joi.object({

src/nni_manager/training_service/pai/paiConfig.ts

@@ -34,6 +34,8 @@ export class PAITaskRole {
     public readonly gpuNumber: number;
     // Executable command for tasks in the task role, can not be empty
     public readonly command: string;
+    //Shared memory for one task in the task role
+    public readonly shmMB?: number;
     
     /**
      * Constructor
@@ -44,13 +46,14 @@ export class PAITaskRole {
      * @param gpuNumber GPU number for one task in the task role, no less than 0
      * @param command Executable command for tasks in the task role, can not be empty
      */
-    constructor(name : string, taskNumber : number, cpuNumber : number, memoryMB : number, gpuNumber : number, command : string) {
+    constructor(name : string, taskNumber : number, cpuNumber : number, memoryMB : number, gpuNumber : number, command : string, shmMB?: number) {
         this.name = name;
         this.taskNumber = taskNumber;
         this.cpuNumber = cpuNumber;
         this.memoryMB = memoryMB;
         this.gpuNumber = gpuNumber;
         this.command = command;    
+        this.shmMB = shmMB;
     }
 }
 
@@ -119,9 +122,11 @@ export class NNIPAITrialConfig extends TrialConfig{
 
     //The virtual cluster job runs on. If omitted, the job will run on default virtual cluster
     public virtualCluster?: string;
+    //Shared memory for one task in the task role
+    public shmMB?: number;
 
     constructor(command : string, codeDir : string, gpuNum : number, cpuNum: number, memoryMB: number, 
-            image: string, dataDir: string, outputDir: string, virtualCluster?: string) {
+            image: string, dataDir: string, outputDir: string, virtualCluster?: string, shmMB?: number) {
         super(command, codeDir, gpuNum);
         this.cpuNum = cpuNum;
         this.memoryMB = memoryMB;
@@ -129,6 +134,7 @@ export class NNIPAITrialConfig extends TrialConfig{
         this.dataDir = dataDir;
         this.outputDir = outputDir;
         this.virtualCluster = virtualCluster;
+        this.shmMB = shmMB;
     }
 }
 

src/nni_manager/training_service/pai/paiTrainingService.ts

@@ -242,7 +242,9 @@ class PAITrainingService implements TrainingService {
                                     // Task GPU number
                                     this.paiTrialConfig.gpuNum, 
                                     // Task command
-                                    nniPaiTrialCommand)];
+                                    nniPaiTrialCommand,
+                                    // Task shared memory
+                                    this.paiTrialConfig.shmMB)];
 
         const paiJobConfig : PAIJobConfig = new PAIJobConfig(
                                     // Job name

src/sdk/pynni/nni/common.py

@@ -34,12 +34,20 @@ def _load_env_args():
         'trial_job_id': os.environ.get('NNI_TRIAL_JOB_ID'),
         'log_dir': os.environ.get('NNI_LOG_DIRECTORY'),
         'role': os.environ.get('NNI_ROLE'),
+        'log_level': os.environ.get('NNI_LOG_LEVEL')
     }
     return namedtuple('EnvArgs', args.keys())(**args)
 
 env_args = _load_env_args()
 '''Arguments passed from environment'''
 
+logLevelMap = {
+    'fatal': logging.FATAL,
+    'error': logging.ERROR,
+    'warning': logging.WARNING,
+    'info': logging.INFO,
+    'debug': logging.DEBUG
+}
 
 _time_format = '%m/%d/%Y, %I:%M:%S %P'
 class _LoggerFileWrapper(TextIOBase):
@@ -53,7 +61,6 @@ class _LoggerFileWrapper(TextIOBase):
             self.file.flush()
         return len(s)
 
-
 def init_logger(logger_file_path):
     """Initialize root logger.
     This will redirect anything from logging.getLogger() as well as stdout to specified file.
@@ -63,6 +70,12 @@ def init_logger(logger_file_path):
         logger_file_path = 'unittest.log'
     elif env_args.log_dir is not None:
         logger_file_path = os.path.join(env_args.log_dir, logger_file_path)
+    if env_args.log_level and logLevelMap.get(env_args.log_level):
+        log_level = logLevelMap[env_args.log_level]
+    else:
+        log_level = logging.INFO #default log level is INFO
+
+    
     logger_file = open(logger_file_path, 'w')
     fmt = '[%(asctime)s] %(levelname)s (%(name)s/%(threadName)s) %(message)s'
     logging.Formatter.converter = time.localtime
@@ -72,10 +85,10 @@ def init_logger(logger_file_path):
 
     root_logger = logging.getLogger()
     root_logger.addHandler(handler)
-    root_logger.setLevel(logging.DEBUG)
+    root_logger.setLevel(log_level)
 
     # these modules are too verbose
-    logging.getLogger('matplotlib').setLevel(logging.INFO)
+    logging.getLogger('matplotlib').setLevel(log_level)
 
     sys.stdout = _LoggerFileWrapper(logger_file)
 

test/pipelines-it-kubeflow.yml

@@ -43,9 +43,9 @@ jobs:
       python3 -m pip install torch==0.4.1 --user
       python3 -m pip install torchvision==0.2.1 --user
       python3 -m pip install keras==2.1.6 --user
-      python3 -m pip install tensorflow-gpu==1.10.0 --user
+      python3 -m pip install tensorflow==1.12.0 --user
       sudo apt-get install swig -y
-      nnictl package install --name=SMAC
+      PATH=$HOME/.local/bin:$PATH nnictl package install --name=SMAC
     displayName: 'Install dependencies for integration tests in Kubeflow mode'
 
   - script: |

test/pipelines-it-local.yml

@@ -12,9 +12,9 @@ jobs:
       python3 -m pip install torch==0.4.1 --user
       python3 -m pip install torchvision==0.2.1 --user
       python3 -m pip install keras==2.1.6 --user
-      python3 -m pip install tensorflow-gpu==1.10.0 --user
+      python3 -m pip install tensorflow-gpu==1.12.0 --user
       sudo apt-get install swig -y
-      nnictl package install --name=SMAC
+      PATH=$HOME/.local/bin:$PATH nnictl package install --name=SMAC
     displayName: 'Install dependencies for integration tests'
   - script: |
       cd test

test/pipelines-it-pai.yml

@@ -43,9 +43,9 @@ jobs:
       python3 -m pip install torch==0.4.1 --user
       python3 -m pip install torchvision==0.2.1 --user
       python3 -m pip install keras==2.1.6 --user
-      python3 -m pip install tensorflow-gpu==1.10.0 --user
+      python3 -m pip install tensorflow-gpu==1.12.0 --user
       sudo apt-get install swig -y
-      nnictl package install --name=SMAC
+      PATH=$HOME/.local/bin:$PATH nnictl package install --name=SMAC
     displayName: 'Install dependencies for integration tests in PAI mode'
 
   - script: |

test/pipelines-it-remote.yml

@@ -2,11 +2,15 @@ jobs:
 - job: 'integration_test_remote'
 
   steps:
-  - script: python3 -m pip install --upgrade pip setuptools
+  - script: python3 -m pip install --upgrade pip setuptools --user
     displayName: 'Install python tools'
   - script: |
       source install.sh
     displayName: 'Install nni toolkit via source code'
+  - script: |
+      sudo apt-get install swig -y
+      PATH=$HOME/.local/bin:$PATH nnictl package install --name=SMAC
+    displayName: 'Install dependencies for integration tests in remote mode'
   - task: CopyFilesOverSSH@0
     inputs:
       sshEndpoint: remote_nni-ci-gpu-01

tools/nni_cmd/config_schema.py

@@ -130,6 +130,7 @@ pai_trial_schema = {
     'cpuNum': And(int, lambda x: 0 <= x <= 99999),
     'memoryMB': int,
     'image': str,
+    Optional('shmMB'): int,
     Optional('dataDir'): Regex(r'hdfs://(([0-9]{1,3}.){3}[0-9]{1,3})(:[0-9]{2,5})?(/.*)?'),
     Optional('outputDir'): Regex(r'hdfs://(([0-9]{1,3}.){3}[0-9]{1,3})(:[0-9]{2,5})?(/.*)?'),
     Optional('virtualCluster'): str

tools/nni_cmd/launcher_utils.py

@@ -215,10 +215,19 @@ def validate_machine_list(experiment_config):
         print_error('Please set machineList!')
         exit(1)
 
+def validate_pai_trial_conifg(experiment_config):
+    '''validate the trial config in pai platform'''
+    if experiment_config.get('trainingServicePlatform') == 'pai':
+        if experiment_config.get('trial').get('shmMB') and \
+        experiment_config['trial']['shmMB'] > experiment_config['trial']['memoryMB']:
+            print_error('shmMB should be no more than memoryMB!')
+            exit(1)
+
 def validate_all_content(experiment_config, config_path):
     '''Validate whether experiment_config is valid'''
     parse_path(experiment_config, config_path)
     validate_common_content(experiment_config)
+    validate_pai_trial_conifg(experiment_config)
     experiment_config['maxExecDuration'] = parse_time(experiment_config['maxExecDuration'])
     if experiment_config.get('advisor'):
         parse_advisor_content(experiment_config)