azureStorageClientUtils.ts 8.95 KB
Newer Older
liuzhe-lz's avatar
liuzhe-lz committed
1
2
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
SparkSnail's avatar
SparkSnail committed
3

4
5
6
import azureStorage from 'azure-storage';
import fs from 'fs';
import path from 'path';
SparkSnail's avatar
SparkSnail committed
7
import { Deferred } from 'ts-deferred';
8
import { String } from 'typescript-string-operations';
9
10
import { getLogger } from 'common/log';
import { mkDirP } from 'common/utils';
SparkSnail's avatar
SparkSnail committed
11
12

export namespace AzureStorageClientUtility {
13

SparkSnail's avatar
SparkSnail committed
14
15
    /**
     * create azure share
16
17
     * @param fileServerClient
     * @param azureShare
SparkSnail's avatar
SparkSnail committed
18
     */
19
20
    export async function createShare(fileServerClient: any, azureShare: any): Promise<boolean> {
        const deferred: Deferred<boolean> = new Deferred<boolean>();
21
        fileServerClient.createShareIfNotExists(azureShare, (error: any, _result: any, _response: any) => {
22
            if (error) {
liuzhe-lz's avatar
liuzhe-lz committed
23
                getLogger('AzureStorageClientUtility')
24
                  .error(`Create share failed:, ${error}`);
25
                deferred.resolve(false);
26
            } else {
27
                deferred.resolve(true);
SparkSnail's avatar
SparkSnail committed
28
            }
29
30
        });

SparkSnail's avatar
SparkSnail committed
31
32
        return deferred.promise;
    }
33

SparkSnail's avatar
SparkSnail committed
34
35
    /**
     * Create a new directory (NOT recursively) in azure file storage.
36
37
38
     * @param fileServerClient
     * @param azureFoler
     * @param azureShare
SparkSnail's avatar
SparkSnail committed
39
     */
40
41
    export async function createDirectory(fileServerClient: azureStorage.FileService, azureFoler: any, azureShare: any): Promise<boolean> {
        const deferred: Deferred<boolean> = new Deferred<boolean>();
42
        fileServerClient.createDirectoryIfNotExists(azureShare, azureFoler, (error: any, _result: any, _response: any) => {
43
            if (error) {
liuzhe-lz's avatar
liuzhe-lz committed
44
                getLogger('AzureStorageClientUtility')
45
                  .error(`Create directory failed:, ${error}`);
46
                deferred.resolve(false);
47
            } else {
48
                deferred.resolve(true);
SparkSnail's avatar
SparkSnail committed
49
            }
50
        });
SparkSnail's avatar
SparkSnail committed
51
52
53
54
55
56
        return deferred.promise;
    }

    /**
     * Create a new directory recursively in azure file storage
     * @param fileServerClient
57
     * @param azureDirectory
SparkSnail's avatar
SparkSnail committed
58
     */
59
    export async function createDirectoryRecursive(fileServerClient: azureStorage.FileService, azureDirectory: string,
60
61
                                                   azureShare: any): Promise<boolean> {
        const deferred: Deferred<boolean> = new Deferred<boolean>();
62
63
64
        const directories: string[] = azureDirectory.split('/');
        let rootDirectory: string = '';
        for (const directory of directories) {
SparkSnail's avatar
SparkSnail committed
65
            rootDirectory += directory;
chicm-ms's avatar
chicm-ms committed
66
            const result: boolean = await createDirectory(fileServerClient, rootDirectory, azureShare);
67
68
69
70
            if (!result) {
                deferred.resolve(false);
                return deferred.promise;
            }
SparkSnail's avatar
SparkSnail committed
71
72
            rootDirectory += '/';
        }
73
        deferred.resolve(true);
74

SparkSnail's avatar
SparkSnail committed
75
76
        return deferred.promise;
    }
77

SparkSnail's avatar
SparkSnail committed
78
79
    /**
     * upload a file to azure storage
80
81
82
83
84
     * @param fileServerClient
     * @param azureDirectory
     * @param azureFileName
     * @param azureShare
     * @param localFilePath
SparkSnail's avatar
SparkSnail committed
85
     */
86
    async function uploadFileToAzure(fileServerClient: any, azureDirectory: string, azureFileName: any, azureShare: any,
87
88
                                     localFilePath: string): Promise<boolean> {
        const deferred: Deferred<boolean> = new Deferred<boolean>();
89
        await fileServerClient.createFileFromLocalFile(azureShare, azureDirectory, azureFileName, localFilePath,
90
                                                       (error: any, _result: any, _response: any) => {
91
            if (error) {
liuzhe-lz's avatar
liuzhe-lz committed
92
                getLogger('AzureStorageClientUtility')
93
                  .error(`Upload file failed:, ${error}`);
94
                deferred.resolve(false);
95
            } else {
96
                deferred.resolve(true);
SparkSnail's avatar
SparkSnail committed
97
            }
98
99
        });

SparkSnail's avatar
SparkSnail committed
100
101
        return deferred.promise;
    }
102

SparkSnail's avatar
SparkSnail committed
103
104
    /**
     * download a file from azure storage
105
106
107
108
109
     * @param fileServerClient
     * @param azureDirectory
     * @param azureFileName
     * @param azureShare
     * @param localFilePath
SparkSnail's avatar
SparkSnail committed
110
     */
111
    async function downloadFile(fileServerClient: any, azureDirectory: string, azureFileName: any, azureShare: any,
112
113
                                localFilePath: string): Promise<boolean> {
        const deferred: Deferred<boolean> = new Deferred<boolean>();
114
        await fileServerClient.getFileToStream(azureShare, azureDirectory, azureFileName, fs.createWriteStream(localFilePath),
115
                                               (error: any, _result: any, _response: any) => {
116
            if (error) {
liuzhe-lz's avatar
liuzhe-lz committed
117
                getLogger('AzureStorageClientUtility')
118
                  .error(`Download file failed:, ${error}`);
119
                deferred.resolve(false);
120
            } else {
121
                deferred.resolve(true);
SparkSnail's avatar
SparkSnail committed
122
            }
123
124
        });

SparkSnail's avatar
SparkSnail committed
125
126
127
128
129
130
131
132
133
134
        return deferred.promise;
    }

    /**
     * Upload a directory to azure file storage
     * @param fileServerClient : the client of file server
     * @param azureDirectory : the directory in azure file storage
     * @param azureShare : the azure share used
     * @param localDirectory : local directory to be uploaded
     */
135
    export async function uploadDirectory(fileServerClient: azureStorage.FileService, azureDirectory: string, azureShare: any,
136
137
                                          localDirectory: string): Promise<boolean> {
        const deferred: Deferred<boolean> = new Deferred<boolean>();
SparkSnail's avatar
SparkSnail committed
138
        const fileNameArray: string[] = fs.readdirSync(localDirectory);
chicm-ms's avatar
chicm-ms committed
139
        const result: boolean = await createDirectoryRecursive(fileServerClient, azureDirectory, azureShare);
140
141
142
143
        if (!result) {
            deferred.resolve(false);
            return deferred.promise;
        }
144
        for (const fileName of fileNameArray) {
SparkSnail's avatar
SparkSnail committed
145
146
            const fullFilePath: string = path.join(localDirectory, fileName);
            try {
147
148
                let resultUploadFile: boolean = true;
                let resultUploadDir: boolean = true;
149
150
                if (fs.lstatSync(fullFilePath)
                      .isFile()) {
151
                    resultUploadFile = await uploadFileToAzure(fileServerClient, azureDirectory, fileName, azureShare, fullFilePath);
SparkSnail's avatar
SparkSnail committed
152
153
                } else {
                    // If filePath is a directory, recuisively copy it to azure
154
155
156
157
158
                    resultUploadDir = await uploadDirectory(fileServerClient, String.Format('{0}/{1}', azureDirectory, fileName), azureShare, fullFilePath);
                }
                if (!(resultUploadFile && resultUploadDir)) {
                    deferred.resolve(false);
                    return deferred.promise;
SparkSnail's avatar
SparkSnail committed
159
                }
160
            } catch (error) {
161
                deferred.resolve(false);
162

SparkSnail's avatar
SparkSnail committed
163
164
165
166
                return deferred.promise;
            }
        }
        // All files/directories are copied successfully, resolve
167
        deferred.resolve(true);
168

SparkSnail's avatar
SparkSnail committed
169
170
        return deferred.promise;
    }
171

SparkSnail's avatar
SparkSnail committed
172
173
    /**
     * downlod a directory from azure
174
175
176
177
     * @param fileServerClient
     * @param azureDirectory
     * @param azureShare
     * @param localDirectory
SparkSnail's avatar
SparkSnail committed
178
     */
179
180
    export async function downloadDirectory(fileServerClient: any, azureDirectory: string, azureShare: any, localDirectory: string):
     Promise<void> {
SparkSnail's avatar
SparkSnail committed
181
        const deferred: Deferred<void> = new Deferred<void>();
182
183
        await mkDirP(localDirectory);
        fileServerClient.listFilesAndDirectoriesSegmented(azureShare, azureDirectory, 'null',
184
                                                          async (_error: any, result: any, _response: any) => {
185
            if (('entries' in result) === false) {
liuzhe-lz's avatar
liuzhe-lz committed
186
                getLogger('AzureStorageClientUtility')
187
                  .error(`list files failed, can't get entries in result`);
SparkSnail's avatar
SparkSnail committed
188
189
190
                throw new Error(`list files failed, can't get entries in result`);
            }

191
            if (('files' in result.entries) === false) {
liuzhe-lz's avatar
liuzhe-lz committed
192
                getLogger('AzureStorageClientUtility')
193
                  .error(`list files failed, can't get files in result['entries']`);
SparkSnail's avatar
SparkSnail committed
194
195
196
                throw new Error(`list files failed, can't get files in result['entries']`);
            }

197
            if (('directories' in result.directories) === false) {
liuzhe-lz's avatar
liuzhe-lz committed
198
                getLogger('AzureStorageClientUtility')
199
                  .error(`list files failed, can't get directories in result['entries']`);
SparkSnail's avatar
SparkSnail committed
200
201
202
                throw new Error(`list files failed, can't get directories in result['entries']`);
            }

203
            for (const fileName of result.entries.files) {
SparkSnail's avatar
SparkSnail committed
204
                const fullFilePath: string = path.join(localDirectory, fileName.name);
205
                await downloadFile(fileServerClient, azureDirectory, fileName.name, azureShare, fullFilePath);
SparkSnail's avatar
SparkSnail committed
206
            }
207

208
209
210
211
            for (const directoryName of result.entries.directories) {
                const fullDirectoryPath: string = path.join(localDirectory, directoryName.name);
                const fullAzureDirectory: string = path.join(azureDirectory, directoryName.name);
                await downloadDirectory(fileServerClient, fullAzureDirectory, azureShare, fullDirectoryPath);
SparkSnail's avatar
SparkSnail committed
212
213
            }
            deferred.resolve();
214
215
        });

SparkSnail's avatar
SparkSnail committed
216
217
218
        return deferred.promise;
    }
}