[WIP] Enable optional Pod Spec for FrameworkController platform (#3379)

.gitignore

@@ -11,7 +11,6 @@
 /ts/nni_manager/metrics.json
 /ts/nni_manager/trial_jobs.json
 
-
 # Logs
 logs
 *.log

docs/en_US/TrainingService/FrameworkControllerMode.rst

@@ -28,6 +28,16 @@ Prerequisite for Azure Kubernetes Service
 #. Follow the `guideline <https://docs.microsoft.com/en-us/azure/storage/common/storage-quickstart-create-account?tabs=portal>`__ to create azure file storage account. If you use Azure Kubernetes Service, NNI need Azure Storage Service to store code files and the output files.
 #. To access Azure storage service, NNI need the access key of the storage account, and NNI uses `Azure Key Vault <https://azure.microsoft.com/en-us/services/key-vault/>`__ Service to protect your private key. Set up Azure Key Vault Service, add a secret to Key Vault to store the access key of Azure storage account. Follow this `guideline <https://docs.microsoft.com/en-us/azure/key-vault/quick-create-cli>`__ to store the access key.
 
+
+Prerequisite for PVC storage mode
+-----------------------------------------
+In order to use persistent volume claims instead of NFS or Azure storage, related storage must
+be created manually, in the namespace your trials will run later. This restriction is due to the
+fact, that persistent volume claims are hard to recycle and thus can quickly mess with a cluster's
+storage management. Persistent volume claims can be created by e.g. using kubectl. Please refer
+to the official Kubernetes documentation for `further information <https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims>`__.
+
+
 Setup FrameworkController
 -------------------------
 
@@ -116,6 +126,37 @@ Trial configuration in frameworkcontroller mode have the following configuration
   * image: the docker image used to create pod and run the program.
   * frameworkAttemptCompletionPolicy: the policy to run framework, please refer the `user-manual <https://github.com/Microsoft/frameworkcontroller/blob/master/doc/user-manual.md#frameworkattemptcompletionpolicy>`__ to get the specific information. Users could use the policy to control the pod, for example, if ps does not stop, only worker stops, The completion policy could helps stop ps.
 
+NNI also offers the possibility to include a customized frameworkcontroller template similar
+to the aforementioned tensorflow example. A valid configuration the may look like:
+
+.. code-block:: yaml
+
+    experimentName: example_mnist_pytorch
+    trialConcurrency: 1
+    maxExecDuration: 1h
+    maxTrialNum: 2
+    logLevel: trace
+    trainingServicePlatform: frameworkcontroller
+    searchSpacePath: search_space.json
+    tuner:
+      builtinTunerName: TPE
+      classArgs:
+        optimize_mode: maximize
+    assessor:
+      builtinAssessorName: Medianstop
+      classArgs:
+        optimize_mode: maximize
+    trial:
+      codeDir: .
+    frameworkcontrollerConfig:
+      configPath: fc_template.yml
+      storage: pvc
+      namespace: twin-pipelines
+      pvc:
+        path: /mnt/data
+
+Note that in this example a persistent volume claim has been used, that must be created manually in the specified namespace beforehand. Stick to the mnist-pytorch example (:githublink: `<examples/trials/mnist-pytorch>`__) for a more detailed config (:githublink: `<examples/trials/mnist-pytorch/config_frameworkcontroller_custom.yml>`__) and frameworkcontroller template (:githublink: `<examples/trials/fc_template.yml>`__).
+
 How to run example
 ------------------
 

examples/trials/mnist-pytorch/config_frameworkcontroller_custom.yml

+authorName: default
+experimentName: example_mnist_pytorch
+trialConcurrency: 1
+maxExecDuration: 1h
+maxTrialNum: 10
+logLevel: trace
+#choice: local, remote, pai, kubeflow
+trainingServicePlatform: frameworkcontroller
+searchSpacePath: search_space.json
+#choice: true, false
+useAnnotation: false
+tuner:
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner, GPTuner
+  builtinTunerName: TPE
+  classArgs:
+    #choice: maximize, minimize
+    optimize_mode: maximize
+assessor:
+  builtinAssessorName: Medianstop
+  classArgs:
+    optimize_mode: maximize
+trial:
+  codeDir: .
+frameworkcontrollerConfig:
+  configPath: fc_template.yml
+  storage: pvc
+  namespace: "default"
+  pvc:
+    path: "/tmp/mount"

examples/trials/mnist-pytorch/fc_template.yml

+apiVersion: frameworkcontroller.microsoft.com/v1
+kind: Framework
+metadata:
+  name: pytorchcpu
+  namespace: default
+spec:
+  executionType: Start
+  retryPolicy:
+    fancyRetryPolicy: true
+    maxRetryCount: 2
+  taskRoles:
+  - name: worker
+    taskNumber: 1
+    frameworkAttemptCompletionPolicy:
+      minFailedTaskCount: 1
+      minSucceededTaskCount: 3
+    task:
+      retryPolicy:
+        fancyRetryPolicy: false
+        maxRetryCount: 0
+      podGracefulDeletionTimeoutSec: 1800
+      pod:
+        spec:
+          restartPolicy: Never
+          hostNetwork: false
+          containers:
+          - name: mnist-pytorch
+            image: msranni/nni:latest
+            command: ["python", "mnist.py"]
+            ports:
+            - containerPort: 5001
+            volumeMounts:
+            - name: frameworkbarrier-volume
+              mountPath: /mnt/frameworkbarrier
+            - name: data-volume
+              mountPath: /tmp/mount
+          serviceAccountName: frameworkbarrier
+          initContainers:
+          - name: frameworkbarrier
+            image: frameworkcontroller/frameworkbarrier
+            volumeMounts:
+            - name: frameworkbarrier-volume
+              mountPath: /mnt/frameworkbarrier
+          volumes:
+          - name: frameworkbarrier-volume
+            emptyDir: {}
+          - name: data-volume
+            persistentVolumeClaim:
+              claimName: nni-storage

nni/tools/nnictl/config_schema.py

@@ -4,11 +4,17 @@
 import json
 import logging
 import os
+
 import netifaces
-from schema import Schema, And, Optional, Regex, Or, SchemaError
-from nni.tools.package_utils import create_validator_instance, get_all_builtin_names, get_registered_algo_meta
-from .constants import SCHEMA_TYPE_ERROR, SCHEMA_RANGE_ERROR, SCHEMA_PATH_ERROR
+from nni.tools.package_utils import (
+    create_validator_instance,
+    get_all_builtin_names,
+    get_registered_algo_meta,
+)
+from schema import And, Optional, Or, Regex, Schema, SchemaError
+
 from .common_utils import get_yml_content, print_warning
+from .constants import SCHEMA_PATH_ERROR, SCHEMA_RANGE_ERROR, SCHEMA_TYPE_ERROR
 
 
 def setType(key, valueType):
@@ -183,9 +189,9 @@ pai_yarn_trial_schema = {
         Optional('virtualCluster'): setType('virtualCluster', str),
         Optional('nasMode'): setChoice('nasMode', 'classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode'),
         Optional('portList'): [{
-            "label": setType('label', str),
-            "beginAt": setType('beginAt', int),
-            "portNumber": setType('portNumber', int)
+            'label': setType('label', str),
+            'beginAt': setType('beginAt', int),
+            'portNumber': setType('portNumber', int)
         }]
     }
 }
@@ -376,7 +382,7 @@ kubeflow_config_schema = {
 frameworkcontroller_trial_schema = {
     'trial': {
         'codeDir':  setPathCheck('codeDir'),
-        'taskRoles': [{
+        Optional('taskRoles'): [{
             'name': setType('name', str),
             'taskNum': setType('taskNum', int),
             'frameworkAttemptCompletionPolicy': {
@@ -395,14 +401,22 @@ frameworkcontroller_trial_schema = {
 
 frameworkcontroller_config_schema = {
     'frameworkcontrollerConfig': Or({
-        Optional('storage'): setChoice('storage', 'nfs', 'azureStorage'),
+        Optional('storage'): setChoice('storage', 'nfs', 'azureStorage', 'pvc'),
         Optional('serviceAccountName'): setType('serviceAccountName', str),
         'nfs': {
             'server': setType('server', str),
             'path': setType('path', str)
-        }
+        },
+        Optional('namespace'): setType('namespace', str),
+        Optional('configPath'): setType('configPath', str),
     }, {
-        Optional('storage'): setChoice('storage', 'nfs', 'azureStorage'),
+        Optional('storage'): setChoice('storage', 'nfs', 'azureStorage', 'pvc'),
+        Optional('serviceAccountName'): setType('serviceAccountName', str),
+        'configPath': setType('configPath', str),
+        'pvc': {'path': setType('server', str)},
+        Optional('namespace'): setType('namespace', str),
+    }, {
+        Optional('storage'): setChoice('storage', 'nfs', 'azureStorage', 'pvc'),
         Optional('serviceAccountName'): setType('serviceAccountName', str),
         'keyVault': {
             'vaultName': And(Regex('([0-9]|[a-z]|[A-Z]|-){1,127}'),
@@ -416,7 +430,9 @@ frameworkcontroller_config_schema = {
             'azureShare': And(Regex('([0-9]|[a-z]|[A-Z]|-){3,63}'),
                               error='ERROR: azureShare format error, azureShare support using (0-9|a-z|A-Z|-)')
         },
-        Optional('uploadRetryCount'): setNumberRange('uploadRetryCount', int, 1, 99999)
+        Optional('uploadRetryCount'): setNumberRange('uploadRetryCount', int, 1, 99999),
+        Optional('namespace'): setType('namespace', str),
+        Optional('configPath'): setType('configPath', str),
     })
 }
 
@@ -479,6 +495,7 @@ class NNIConfigSchema:
         self.validate_kubeflow_operators(experiment_config)
         self.validate_eth0_device(experiment_config)
         self.validate_hybrid_platforms(experiment_config)
+        self.validate_frameworkcontroller_trial_config(experiment_config)
 
     def validate_tuner_adivosr_assessor(self, experiment_config):
         if experiment_config.get('advisor'):
@@ -601,3 +618,24 @@ class NNIConfigSchema:
                 if config_name and not experiment_config.get(config_name):
                     raise SchemaError('Need to set {0} for {1} in hybrid mode!'.format(config_name, platform))
 
+    def validate_frameworkcontroller_trial_config(self, experiment_config):
+        if experiment_config.get('trainingServicePlatform') == 'frameworkcontroller':
+            if not experiment_config.get('trial').get('taskRoles'):
+                if not experiment_config.get('frameworkcontrollerConfig').get('configPath'):
+                    raise SchemaError("""If no taskRoles are specified a valid custom frameworkcontroller config should
+                                         be set using the configPath attribute in frameworkcontrollerConfig!""")
+                config_content = get_yml_content(experiment_config.get('frameworkcontrollerConfig').get('configPath'))
+                if not config_content.get('spec').get('taskRoles') or not len(config_content.get('spec').get('taskRoles')):
+                    raise SchemaError('Invalid frameworkcontroller config! No taskRoles were specified!')
+                if not config_content.get('spec').get('taskRoles')[0].get('task'):
+                    raise SchemaError('Invalid frameworkcontroller config! No task was specified for taskRole!')
+                names = []
+                for taskRole in config_content.get('spec').get('taskRoles'):
+                    if not "name" in taskRole:
+                        raise SchemaError('Invalid frameworkcontroller config! Name is missing for taskRole!')
+                    names.append(taskRole.get("name"))
+                if len(names) > len(set(names)):
+                    raise SchemaError('Invalid frameworkcontroller config! Duplicate taskrole names!')
+                if not config_content.get('metadata').get('name'):
+                    raise SchemaError('Invalid frameworkcontroller config! No experiment name was specified!')
+

nni/tools/nnictl/launcher_utils.py

@@ -100,6 +100,11 @@ def parse_path(experiment_config, config_path):
     if experiment_config['trial'].get('paiConfigPath'):
         parse_relative_path(root_path, experiment_config['trial'], 'paiConfigPath')
 
+    # For frameworkcontroller a custom configuration path may be specified
+    if experiment_config.get('frameworkcontrollerConfig'):
+        if experiment_config['frameworkcontrollerConfig'].get('configPath'):
+            parse_relative_path(root_path, experiment_config['frameworkcontrollerConfig'], 'configPath')
+
 def set_default_values(experiment_config):
     if experiment_config.get('maxExecDuration') is None:
         experiment_config['maxExecDuration'] = '999d'

ts/nni_manager/rest_server/restValidationSchemas.ts

@@ -152,6 +152,10 @@ export namespace ValidationSchemas {
             frameworkcontroller_config: joi.object({ // eslint-disable-line @typescript-eslint/camelcase
                 storage: joi.string().min(1),
                 serviceAccountName: joi.string().min(1),
+                pvc: joi.object({
+                    path: joi.string().min(1).required()
+                }),
+                configPath: joi.string().min(1),
                 nfs: joi.object({
                     server: joi.string().min(1).required(),
                     path: joi.string().min(1).required()
@@ -164,7 +168,8 @@ export namespace ValidationSchemas {
                     accountName: joi.string().regex(/^([0-9]|[a-z]|[A-Z]|-){3,31}$/),
                     azureShare: joi.string().regex(/^([0-9]|[a-z]|[A-Z]|-){3,63}$/)
                 }),
-                uploadRetryCount: joi.number().min(1)
+                uploadRetryCount: joi.number().min(1),
+                namespace: joi.string().min(1)
             }),
             dlts_config: joi.object({ // eslint-disable-line @typescript-eslint/camelcase
                 dashboard: joi.string().min(1),

ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerApiClient.ts

@@ -4,7 +4,7 @@
 'use strict';
 
 import * as fs from 'fs';
-import { GeneralK8sClient, KubernetesCRDClient } from '../kubernetesApiClient';
+import {GeneralK8sClient, KubernetesCRDClient} from '../kubernetesApiClient';
 
 /**
  * FrameworkController ClientV1
@@ -13,14 +13,16 @@ class FrameworkControllerClientV1 extends KubernetesCRDClient {
     /**
      * constructor, to initialize frameworkcontroller CRD definition
      */
-    public constructor() {
+    public namespace: string;
+    public constructor(namespace?: string) {
         super();
+        this.namespace = namespace ? namespace : "default"
         this.crdSchema = JSON.parse(fs.readFileSync('./config/frameworkcontroller/frameworkcontrollerjob-crd-v1.json', 'utf8'));
         this.client.addCustomResourceDefinition(this.crdSchema);
     }
 
     protected get operator(): any {
-        return this.client.apis['frameworkcontroller.microsoft.com'].v1.namespaces('default').frameworks;
+        return this.client.apis['frameworkcontroller.microsoft.com'].v1.namespaces(this.namespace).frameworks;
     }
 
     public get containerName(): string {
@@ -35,9 +37,9 @@ class FrameworkControllerClientFactory {
     /**
      * Factory method to generate operator client
      */
-    public static createClient(): KubernetesCRDClient {
-        return new FrameworkControllerClientV1();
+    public static createClient(namespace?: string): KubernetesCRDClient {
+        return new FrameworkControllerClientV1(namespace);
     }
 }
 
-export { FrameworkControllerClientFactory, GeneralK8sClient };
+export {FrameworkControllerClientFactory, GeneralK8sClient};

ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerConfig.ts

@@ -5,8 +5,10 @@
 
 import * as assert from 'assert';
 
-import { AzureStorage, KeyVaultConfig, KubernetesClusterConfig, KubernetesClusterConfigAzure, KubernetesClusterConfigNFS,
-    KubernetesStorageKind, KubernetesTrialConfig, KubernetesTrialConfigTemplate, NFSConfig, StorageConfig
+import {
+    AzureStorage, KeyVaultConfig, KubernetesClusterConfig, KubernetesClusterConfigAzure, KubernetesClusterConfigNFS,
+    KubernetesStorageKind, KubernetesTrialConfig, KubernetesTrialConfigTemplate, NFSConfig, StorageConfig, KubernetesClusterConfigPVC,
+    PVCConfig,
 } from '../kubernetesConfig';
 
 export class FrameworkAttemptCompletionPolicy {
@@ -47,60 +49,97 @@ export class FrameworkControllerTrialConfig extends KubernetesTrialConfig {
 
 export class FrameworkControllerClusterConfig extends KubernetesClusterConfig {
     public readonly serviceAccountName: string;
-    constructor(apiVersion: string, serviceAccountName: string) {
-        super(apiVersion);
+    constructor(apiVersion: string, serviceAccountName: string, configPath?: string, namespace?: string) {
+        super(apiVersion, undefined, namespace);
         this.serviceAccountName = serviceAccountName;
     }
 }
 
+export class FrameworkControllerClusterConfigPVC extends KubernetesClusterConfigPVC {
+    public readonly serviceAccountName: string;
+    public readonly configPath: string;
+    constructor(serviceAccountName: string, apiVersion: string, pvc: PVCConfig, configPath: string,
+        storage?: KubernetesStorageKind, namespace?: string) {
+        super(apiVersion, pvc, storage, namespace);
+        this.serviceAccountName = serviceAccountName;
+        this.configPath = configPath
+    }
+
+    public static getInstance(jsonObject: object): FrameworkControllerClusterConfigPVC {
+        const kubernetesClusterConfigObjectPVC: FrameworkControllerClusterConfigPVC = <FrameworkControllerClusterConfigPVC>jsonObject;
+        assert(kubernetesClusterConfigObjectPVC !== undefined);
+
+        return new FrameworkControllerClusterConfigPVC(
+            kubernetesClusterConfigObjectPVC.serviceAccountName,
+            kubernetesClusterConfigObjectPVC.apiVersion,
+            kubernetesClusterConfigObjectPVC.pvc,
+            kubernetesClusterConfigObjectPVC.configPath,
+            kubernetesClusterConfigObjectPVC.storage,
+            kubernetesClusterConfigObjectPVC.namespace
+        );
+    }
+}
+
 export class FrameworkControllerClusterConfigNFS extends KubernetesClusterConfigNFS {
     public readonly serviceAccountName: string;
+    public readonly configPath?: string;
     constructor(
         serviceAccountName: string,
         apiVersion: string,
         nfs: NFSConfig,
-            storage?: KubernetesStorageKind
+        storage?: KubernetesStorageKind,
+        namespace?: string,
+        configPath?: string
     ) {
-        super(apiVersion, nfs, storage);
+        super(apiVersion, nfs, storage, namespace);
         this.serviceAccountName = serviceAccountName;
+        this.configPath = configPath
     }
 
     public static getInstance(jsonObject: object): FrameworkControllerClusterConfigNFS {
-        const kubeflowClusterConfigObjectNFS: FrameworkControllerClusterConfigNFS = <FrameworkControllerClusterConfigNFS>jsonObject;
-        assert (kubeflowClusterConfigObjectNFS !== undefined);
+        const kubernetesClusterConfigObjectNFS: FrameworkControllerClusterConfigNFS = <FrameworkControllerClusterConfigNFS>jsonObject;
+        assert(kubernetesClusterConfigObjectNFS !== undefined);
 
         return new FrameworkControllerClusterConfigNFS(
-            kubeflowClusterConfigObjectNFS.serviceAccountName,
-            kubeflowClusterConfigObjectNFS.apiVersion,
-            kubeflowClusterConfigObjectNFS.nfs,
-            kubeflowClusterConfigObjectNFS.storage
+            kubernetesClusterConfigObjectNFS.serviceAccountName,
+            kubernetesClusterConfigObjectNFS.apiVersion,
+            kubernetesClusterConfigObjectNFS.nfs,
+            kubernetesClusterConfigObjectNFS.storage,
+            kubernetesClusterConfigObjectNFS.namespace
         );
     }
 }
 
 export class FrameworkControllerClusterConfigAzure extends KubernetesClusterConfigAzure {
     public readonly serviceAccountName: string;
+    public readonly configPath?: string;
 
     constructor(
         serviceAccountName: string,
         apiVersion: string,
         keyVault: KeyVaultConfig,
         azureStorage: AzureStorage,
-            storage?: KubernetesStorageKind
+        storage?: KubernetesStorageKind,
+        uploadRetryCount?: number,
+        namespace?: string,
+        configPath?: string
     ) {
-        super(apiVersion, keyVault, azureStorage, storage);
+        super(apiVersion, keyVault, azureStorage, storage, uploadRetryCount, namespace);
         this.serviceAccountName = serviceAccountName;
+        this.configPath = configPath
     }
 
     public static getInstance(jsonObject: object): FrameworkControllerClusterConfigAzure {
-        const kubeflowClusterConfigObjectAzure: FrameworkControllerClusterConfigAzure = <FrameworkControllerClusterConfigAzure>jsonObject;
+        const kubernetesClusterConfigObjectAzure: FrameworkControllerClusterConfigAzure = <FrameworkControllerClusterConfigAzure>jsonObject;
 
         return new FrameworkControllerClusterConfigAzure(
-            kubeflowClusterConfigObjectAzure.serviceAccountName,
-            kubeflowClusterConfigObjectAzure.apiVersion,
-            kubeflowClusterConfigObjectAzure.keyVault,
-            kubeflowClusterConfigObjectAzure.azureStorage,
-            kubeflowClusterConfigObjectAzure.storage
+            kubernetesClusterConfigObjectAzure.serviceAccountName,
+            kubernetesClusterConfigObjectAzure.apiVersion,
+            kubernetesClusterConfigObjectAzure.keyVault,
+            kubernetesClusterConfigObjectAzure.azureStorage,
+            kubernetesClusterConfigObjectAzure.storage,
+            kubernetesClusterConfigObjectAzure.uploadRetryCount,
+            kubernetesClusterConfigObjectAzure.namespace
         );
     }
 }
@@ -116,6 +155,8 @@ export class FrameworkControllerClusterConfigFactory {
             return FrameworkControllerClusterConfigAzure.getInstance(jsonObject);
         } else if (storageConfig.storage === undefined || storageConfig.storage === 'nfs') {
             return FrameworkControllerClusterConfigNFS.getInstance(jsonObject);
+        } else if (storageConfig.storage !== undefined && storageConfig.storage === 'pvc') {
+            return FrameworkControllerClusterConfigPVC.getInstance(jsonObject);
         }
         throw new Error(`Invalid json object ${jsonObject}`);
     }

ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts

@@ -202,8 +202,8 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber
             const azureKubeflowClusterConfig: KubeflowClusterConfigAzure = <KubeflowClusterConfigAzure>this.kubeflowClusterConfig;
             return await this.uploadFolderToAzureStorage(srcDirectory, destDirectory, azureKubeflowClusterConfig.uploadRetryCount);
         } else if (this.kubeflowClusterConfig.storage === 'nfs' || this.kubeflowClusterConfig.storage === undefined) {
-            await cpp.exec(`mkdir -p ${this.trialLocalNFSTempFolder}/${destDirectory}`);
-            await cpp.exec(`cp -r ${srcDirectory}/* ${this.trialLocalNFSTempFolder}/${destDirectory}/.`);
+            await cpp.exec(`mkdir -p ${this.trialLocalTempFolder}/${destDirectory}`);
+            await cpp.exec(`cp -r ${srcDirectory}/* ${this.trialLocalTempFolder}/${destDirectory}/.`);
             const nfsKubeflowClusterConfig: KubeflowClusterConfigNFS = <KubeflowClusterConfigNFS>this.kubeflowClusterConfig;
             const nfsConfig: NFSConfig = nfsKubeflowClusterConfig.nfs;
             return `nfs://${nfsConfig.server}:${destDirectory}`;

ts/nni_manager/training_service/kubernetes/kubernetesApiClient.ts

@@ -4,8 +4,8 @@
 'use strict';
 
 // eslint-disable-next-line @typescript-eslint/camelcase
-import { Client1_10, config } from 'kubernetes-client';
-import { getLogger, Logger } from '../../common/log';
+import {Client1_10, config} from 'kubernetes-client';
+import {getLogger, Logger} from '../../common/log';
 
 /**
  * Generic Kubernetes client, target version >= 1.9
@@ -16,13 +16,16 @@ class GeneralK8sClient {
     protected namespace: string = 'default';
 
     constructor() {
-        this.client = new Client1_10({ config: config.fromKubeconfig(), version: '1.9'});
+        this.client = new Client1_10({config: config.fromKubeconfig(), version: '1.9'});
         this.client.loadSpec();
     }
 
     public set setNamespace(namespace: string) {
         this.namespace = namespace;
     }
+    public get getNamespace(): string {
+        return this.namespace;
+    }
 
     private matchStorageClass(response: any): string {
         const adlSupportedProvisioners: RegExp[] = [
@@ -66,7 +69,7 @@ class GeneralK8sClient {
     public async createDeployment(deploymentManifest: any): Promise<string> {
         let result: Promise<string>;
         const response: any = await this.client.apis.apps.v1.namespaces(this.namespace)
-          .deployments.post({ body: deploymentManifest })
+            .deployments.post({body: deploymentManifest})
         if (response.statusCode && (response.statusCode >= 200 && response.statusCode <= 299)) {
             result = Promise.resolve(response.body.metadata.uid);
         } else {
@@ -136,7 +139,7 @@ abstract class KubernetesCRDClient {
     protected crdSchema: any;
 
     constructor() {
-        this.client = new Client1_10({ config: config.fromKubeconfig() });
+        this.client = new Client1_10({config: config.fromKubeconfig()});
         this.client.loadSpec();
     }
 
@@ -219,4 +222,4 @@ abstract class KubernetesCRDClient {
     }
 }
 
-export { KubernetesCRDClient, GeneralK8sClient };
+export {KubernetesCRDClient, GeneralK8sClient};

ts/nni_manager/training_service/kubernetes/kubernetesConfig.ts

@@ -3,16 +3,18 @@
 
 'use strict';
 
-export type KubernetesStorageKind = 'nfs' | 'azureStorage';
-import { MethodNotImplementedError } from '../../common/errors';
+export type KubernetesStorageKind = 'nfs' | 'azureStorage' | 'pvc';
+import {MethodNotImplementedError} from '../../common/errors';
 
 export abstract class KubernetesClusterConfig {
     public readonly storage?: KubernetesStorageKind;
     public readonly apiVersion: string;
+    public readonly namespace?: string;
 
-    constructor(apiVersion: string, storage?: KubernetesStorageKind) {
+    constructor(apiVersion: string, storage?: KubernetesStorageKind, namespace?: string) {
         this.storage = storage;
         this.apiVersion = apiVersion;
+        this.namespace = namespace
     }
 
     public get storageType(): KubernetesStorageKind {
@@ -34,9 +36,10 @@ export class KubernetesClusterConfigNFS extends KubernetesClusterConfig {
     constructor(
         apiVersion: string,
         nfs: NFSConfig,
-            storage?: KubernetesStorageKind
+        storage?: KubernetesStorageKind,
+        namespace?: string
     ) {
-        super(apiVersion, storage);
+        super(apiVersion, storage, namespace);
         this.nfs = nfs;
     }
 
@@ -50,7 +53,8 @@ export class KubernetesClusterConfigNFS extends KubernetesClusterConfig {
         return new KubernetesClusterConfigNFS(
             kubernetesClusterConfigObjectNFS.apiVersion,
             kubernetesClusterConfigObjectNFS.nfs,
-            kubernetesClusterConfigObjectNFS.storage
+            kubernetesClusterConfigObjectNFS.storage,
+            kubernetesClusterConfigObjectNFS.namespace
         );
     }
 }
@@ -65,9 +69,11 @@ export class KubernetesClusterConfigAzure extends KubernetesClusterConfig {
         keyVault: KeyVaultConfig,
         azureStorage: AzureStorage,
         storage?: KubernetesStorageKind,
-            uploadRetryCount?: number
+        uploadRetryCount?: number,
+        namespace?: string,
+
     ) {
-        super(apiVersion, storage);
+        super(apiVersion, storage, namespace);
         this.keyVault = keyVault;
         this.azureStorage = azureStorage;
         this.uploadRetryCount = uploadRetryCount;
@@ -85,18 +91,48 @@ export class KubernetesClusterConfigAzure extends KubernetesClusterConfig {
             kubernetesClusterConfigObjectAzure.keyVault,
             kubernetesClusterConfigObjectAzure.azureStorage,
             kubernetesClusterConfigObjectAzure.storage,
-            kubernetesClusterConfigObjectAzure.uploadRetryCount
+            kubernetesClusterConfigObjectAzure.uploadRetryCount,
+            kubernetesClusterConfigObjectAzure.namespace
         );
     }
 }
 
-export class KubernetesClusterConfigFactory {
+export class KubernetesClusterConfigPVC extends KubernetesClusterConfig {
+    public readonly pvc: PVCConfig;
+
+    constructor(
+        apiVersion: string,
+        pvc: PVCConfig,
+        storage?: KubernetesStorageKind,
+        namespace?: string,
+    ) {
+        super(apiVersion, storage, namespace);
+        this.pvc = pvc;
+    }
+
+    public get storageType(): KubernetesStorageKind {
+        return 'pvc';
+    }
 
+    public static getInstance(jsonObject: object): KubernetesClusterConfigPVC {
+        const kubernetesClusterConfigObjectPVC: KubernetesClusterConfigPVC =
+            <KubernetesClusterConfigPVC>jsonObject;
+        return new KubernetesClusterConfigPVC(
+            kubernetesClusterConfigObjectPVC.apiVersion,
+            kubernetesClusterConfigObjectPVC.pvc,
+            kubernetesClusterConfigObjectPVC.storage,
+            kubernetesClusterConfigObjectPVC.namespace
+        );
+    }
+}
+export class KubernetesClusterConfigFactory {
     public static generateKubernetesClusterConfig(jsonObject: object): KubernetesClusterConfig {
         const storageConfig: StorageConfig = <StorageConfig>jsonObject;
         switch (storageConfig.storage) {
             case 'azureStorage':
                 return KubernetesClusterConfigAzure.getInstance(jsonObject);
+            case 'pvc':
+                return KubernetesClusterConfigPVC.getInstance(jsonObject);
             case 'nfs':
             case undefined:
                 return KubernetesClusterConfigNFS.getInstance(jsonObject);
@@ -121,6 +157,18 @@ export class NFSConfig {
     }
 }
 
+/**
+ * PVC configuration to store Kubernetes job related files
+ */
+export class PVCConfig {
+    // Path of the mounted pvc
+    public readonly path: string;
+
+    constructor(path: string) {
+        this.path = path;
+    }
+}
+
 /**
  * KeyVault configuration to store the key of Azure Storage Service
  * Refer https://docs.microsoft.com/en-us/azure/key-vault/key-vault-manage-with-cli2

ts/nni_manager/training_service/kubernetes/kubernetesTrainingService.ts

@@ -7,21 +7,21 @@ import * as cpp from 'child-process-promise';
 import * as path from 'path';
 
 import * as azureStorage from 'azure-storage';
-import { EventEmitter } from 'events';
-import { Base64 } from 'js-base64';
-import { String } from 'typescript-string-operations';
-import { getExperimentId } from '../../common/experimentStartupInfo';
-import { getLogger, Logger } from '../../common/log';
-import { MethodNotImplementedError } from '../../common/errors';
+import {EventEmitter} from 'events';
+import {Base64} from 'js-base64';
+import {String} from 'typescript-string-operations';
+import {getExperimentId} from '../../common/experimentStartupInfo';
+import {getLogger, Logger} from '../../common/log';
+import {MethodNotImplementedError} from '../../common/errors';
 import {
     NNIManagerIpConfig, TrialJobDetail, TrialJobMetric, LogType
 } from '../../common/trainingService';
-import { delay, getExperimentRootDir, getIPV4Address, getJobCancelStatus, getVersion, uniqueString } from '../../common/utils';
-import { AzureStorageClientUtility } from './azureStorageClientUtils';
-import { GeneralK8sClient, KubernetesCRDClient } from './kubernetesApiClient';
-import { KubernetesClusterConfig } from './kubernetesConfig';
-import { kubernetesScriptFormat, KubernetesTrialJobDetail } from './kubernetesData';
-import { KubernetesJobRestServer } from './kubernetesJobRestServer';
+import {delay, getExperimentRootDir, getIPV4Address, getJobCancelStatus, getVersion, uniqueString} from '../../common/utils';
+import {AzureStorageClientUtility} from './azureStorageClientUtils';
+import {GeneralK8sClient, KubernetesCRDClient} from './kubernetesApiClient';
+import {KubernetesClusterConfig} from './kubernetesConfig';
+import {kubernetesScriptFormat, KubernetesTrialJobDetail} from './kubernetesData';
+import {KubernetesJobRestServer} from './kubernetesJobRestServer';
 
 const fs = require('fs');
 
@@ -34,7 +34,7 @@ abstract class KubernetesTrainingService {
     protected readonly metricsEmitter: EventEmitter;
     protected readonly trialJobsMap: Map<string, KubernetesTrialJobDetail>;
     //  experiment root dir in NFS
-    protected readonly trialLocalNFSTempFolder: string;
+    protected readonly trialLocalTempFolder: string;
     protected stopping: boolean = false;
     protected experimentId!: string;
     protected kubernetesRestServerPort?: number;
@@ -57,7 +57,7 @@ abstract class KubernetesTrainingService {
         this.log = getLogger();
         this.metricsEmitter = new EventEmitter();
         this.trialJobsMap = new Map<string, KubernetesTrialJobDetail>();
-        this.trialLocalNFSTempFolder = path.join(getExperimentRootDir(), 'trials-nfs-tmp');
+        this.trialLocalTempFolder = path.join(getExperimentRootDir(), 'trials-nfs-tmp');
         this.experimentId = getExperimentId();
         this.CONTAINER_MOUNT_PATH = '/tmp/mount';
         this.expContainerCodeFolder = path.join(this.CONTAINER_MOUNT_PATH, 'nni', this.experimentId, 'nni-code');
@@ -191,9 +191,9 @@ abstract class KubernetesTrainingService {
 
         // Unmount NFS
         try {
-            await cpp.exec(`sudo umount ${this.trialLocalNFSTempFolder}`);
+            await cpp.exec(`sudo umount ${this.trialLocalTempFolder}`);
         } catch (error) {
-            this.log.error(`Unmount ${this.trialLocalNFSTempFolder} failed, error is ${error}`);
+            this.log.error(`Unmount ${this.trialLocalTempFolder} failed, error is ${error}`);
         }
 
         // Stop kubernetes rest server
@@ -231,13 +231,15 @@ abstract class KubernetesTrainingService {
             //create sotrage secret
             this.azureStorageSecretName = String.Format('nni-secret-{0}', uniqueString(8)
                 .toLowerCase());
+
+            const namespace = this.genericK8sClient.getNamespace ? this.genericK8sClient.getNamespace : "default"
             await this.genericK8sClient.createSecret(
                 {
                     apiVersion: 'v1',
                     kind: 'Secret',
                     metadata: {
                         name: this.azureStorageSecretName,
-                        namespace: 'default',
+                        namespace: namespace,
                         labels: {
                             app: this.NNI_KUBERNETES_TRIAL_LABEL,
                             expId: getExperimentId()
@@ -297,11 +299,11 @@ abstract class KubernetesTrainingService {
         return Promise.resolve(runScript);
     }
     protected async createNFSStorage(nfsServer: string, nfsPath: string): Promise<void> {
-        await cpp.exec(`mkdir -p ${this.trialLocalNFSTempFolder}`);
+        await cpp.exec(`mkdir -p ${this.trialLocalTempFolder}`);
         try {
-            await cpp.exec(`sudo mount ${nfsServer}:${nfsPath} ${this.trialLocalNFSTempFolder}`);
+            await cpp.exec(`sudo mount ${nfsServer}:${nfsPath} ${this.trialLocalTempFolder}`);
         } catch (error) {
-            const mountError: string = `Mount NFS ${nfsServer}:${nfsPath} to ${this.trialLocalNFSTempFolder} failed, error is ${error}`;
+            const mountError: string = `Mount NFS ${nfsServer}:${nfsPath} to ${this.trialLocalTempFolder} failed, error is ${error}`;
             this.log.error(mountError);
 
             return Promise.reject(mountError);
@@ -309,21 +311,35 @@ abstract class KubernetesTrainingService {
 
         return Promise.resolve();
     }
+    protected async createPVCStorage(pvcPath: string): Promise<void> {
+        try {
+            await cpp.exec(`mkdir -p ${pvcPath}`);
+            await cpp.exec(`sudo ln -s ${pvcPath} ${this.trialLocalTempFolder}`);
+        } catch (error) {
+            const linkError: string = `Linking ${pvcPath} to ${this.trialLocalTempFolder} failed, error is ${error}`;
+            this.log.error(linkError);
+
+            return Promise.reject(linkError);
+        }
+
+        return Promise.resolve();
+    }
 
     protected async createRegistrySecret(filePath: string | undefined): Promise<string | undefined> {
-        if(filePath === undefined || filePath === '') {
+        if (filePath === undefined || filePath === '') {
             return undefined;
         }
         const body = fs.readFileSync(filePath).toString('base64');
         const registrySecretName = String.Format('nni-secret-{0}', uniqueString(8)
             .toLowerCase());
+        const namespace = this.genericK8sClient.getNamespace ? this.genericK8sClient.getNamespace : "default"
         await this.genericK8sClient.createSecret(
             {
                 apiVersion: 'v1',
                 kind: 'Secret',
                 metadata: {
                     name: registrySecretName,
-                    namespace: 'default',
+                    namespace: namespace,
                     labels: {
                         app: this.NNI_KUBERNETES_TRIAL_LABEL,
                         expId: getExperimentId()
@@ -349,7 +365,7 @@ abstract class KubernetesTrainingService {
             throw new Error('azureStorageClient is not initialized');
         }
         let retryCount: number = 1;
-        if(uploadRetryCount) {
+        if (uploadRetryCount) {
             retryCount = uploadRetryCount;
         }
         let uploadSuccess: boolean = false;
@@ -378,4 +394,4 @@ abstract class KubernetesTrainingService {
         return Promise.resolve(folderUriInAzure);
     }
 }
-export { KubernetesTrainingService };
+export {KubernetesTrainingService};