add nfs shared storage integration test (#3913)

pipelines/integration-test-remote-l2l.yml

@@ -94,7 +94,10 @@ jobs:
           --remote_pwd $(password_in_docker) \
           --nni_manager_ip $(manager_ip) \
           --azurestoragetoken $(azureblob_token_test) \
-          --nfs_server $(NFS_IP)
+          --nfs_server $(NFS_IP) \
+          --local_mount_point $(LOCAL_MOUNT_POINT) \
+          --remote_mount_point $(REMOTE_MOUNT_POINT) \
+          --exported_directory $(Exported_Directory)
       python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts remote
     displayName: Integration test (reuse mode)
 

test/config/integration_tests.yml

@@ -49,19 +49,13 @@ testCases:
     kwargs:
       rootpath: /tmp/nnimount/testlocalrootpath
 
-# TODO: Enable this case after nfs server is ready
-#- name: shared-storage-remote-nfs
-#  configFile: test/config/sharedstorage_test/config_sharedstorage_remote_nfs.yml
-#  config:
-#    sharedStorage:
-#      localMountPoint: /tmp/nnimount/testlocalrootpath
-#      remoteMountPoint: /tmp/nnimount/testremoterootpath
-#      nfsServer: $(NFS_IP)
-#      exportedDirectory: /home/nni/mnt/
-#  validator:
-#    class: FileExistValidator
-#    kwargs:
-#      rootpath: /tmp/nnimount/testlocalrootpath
+- name: shared-storage-remote-nfs
+  configFile: test/config/sharedstorage_test/config_sharedstorage_remote_nfs.yml
+  trainingService: remote
+  validator:
+    class: FileExistValidator
+    kwargs:
+      rootpath: /tmp/nnimount/testlocalrootpath
 
 - name: sklearn-regression
   configFile: test/config/examples/sklearn-regression.yml

test/nni_test/nnitest/generate_ts_config.py

@@ -78,6 +78,12 @@ def update_training_service_config(args):
             config[args.ts]['sharedStorage']['storageAccountKey'] = args.azurestoragetoken
         if args.nfs_server is not None:
             config[args.ts]['sharedStorage']['nfsServer'] = args.nfs_server
+        if args.local_mount_point is not None:
+            config[args.ts]['sharedStorage']['localMountPoint'] = args.local_mount_point
+        if args.remote_mount_point is not None:
+            config[args.ts]['sharedStorage']['remoteMountPoint'] = args.remote_mount_point
+        if args.exported_directory is not None:
+            config[args.ts]['sharedStorage']['exportedDirectory'] = args.exported_directory
     elif args.ts == 'adl':
         if args.nni_docker_image is not None:
             config[args.ts]['trial']['image'] = args.nni_docker_image
@@ -122,8 +128,12 @@ if __name__ == '__main__':
     parser.add_argument("--config_version", type=str, choices=['v1', 'v2'], default='v1')
     parser.add_argument("--nni_docker_image", type=str)
     parser.add_argument("--nni_manager_ip", type=str)
+    # args for remote with shared storage
     parser.add_argument("--azurestoragetoken", type=str)
     parser.add_argument("--nfs_server", type=str)
+    parser.add_argument("--local_mount_point", type=str)
+    parser.add_argument("--remote_mount_point", type=str)
+    parser.add_argument("--exported_directory", type=str)
     # args for PAI
     parser.add_argument("--pai_user", type=str)
     parser.add_argument("--pai_pwd", type=str)

test/nni_test/nnitest/run_tests.py

@@ -62,6 +62,7 @@ def update_training_service_config(config, training_service, config_file_path, n
             it_ts_config[training_service].get('sharedStorage').pop('storageAccountKey')
         elif str(sharedStorage.get('storageType')).lower() == 'azureblob':
             it_ts_config[training_service].get('sharedStorage').pop('nfsServer')
+            it_ts_config[training_service].get('sharedStorage').pop('exportedDirectory')
         else:
             it_ts_config[training_service].pop('sharedStorage')