azureStorageClientUtils.ts 10 KB
Newer Older
SparkSnail's avatar
SparkSnail committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
/**
 * Copyright (c) Microsoft Corporation
 * All rights reserved.
 *
 * MIT License
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
 * documentation files (the "Software"), to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
 * to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
 * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

'use strict';

22
23
import * as azureStorage from 'azure-storage';
import * as fs from 'fs';
SparkSnail's avatar
SparkSnail committed
24
25
import * as path from 'path';
import { Deferred } from 'ts-deferred';
26
import { String } from 'typescript-string-operations';
SparkSnail's avatar
SparkSnail committed
27
28
29
import { getLogger } from '../../common/log';
import { mkDirP } from '../../common/utils';

30
// tslint:disable: no-redundant-jsdoc no-any no-unsafe-any
SparkSnail's avatar
SparkSnail committed
31
export namespace AzureStorageClientUtility {
32

SparkSnail's avatar
SparkSnail committed
33
34
    /**
     * create azure share
35
36
     * @param fileServerClient
     * @param azureShare
SparkSnail's avatar
SparkSnail committed
37
     */
38
39
    export async function createShare(fileServerClient: any, azureShare: any): Promise<boolean> {
        const deferred: Deferred<boolean> = new Deferred<boolean>();
40
41
42
43
        fileServerClient.createShareIfNotExists(azureShare, (error: any, result: any, response: any) => {
            if (error) {
                getLogger()
                  .error(`Create share failed:, ${error}`);
44
                deferred.resolve(false);
45
            } else {
46
                deferred.resolve(true);
SparkSnail's avatar
SparkSnail committed
47
            }
48
49
        });

SparkSnail's avatar
SparkSnail committed
50
51
        return deferred.promise;
    }
52

SparkSnail's avatar
SparkSnail committed
53
54
    /**
     * Create a new directory (NOT recursively) in azure file storage.
55
56
57
     * @param fileServerClient
     * @param azureFoler
     * @param azureShare
SparkSnail's avatar
SparkSnail committed
58
     */
59
60
    export async function createDirectory(fileServerClient: azureStorage.FileService, azureFoler: any, azureShare: any): Promise<boolean> {
        const deferred: Deferred<boolean> = new Deferred<boolean>();
61
62
63
64
        fileServerClient.createDirectoryIfNotExists(azureShare, azureFoler, (error: any, result: any, response: any) => {
            if (error) {
                getLogger()
                  .error(`Create directory failed:, ${error}`);
65
                deferred.resolve(false);
66
            } else {
67
                deferred.resolve(true);
SparkSnail's avatar
SparkSnail committed
68
            }
69
        });
SparkSnail's avatar
SparkSnail committed
70
71
72
73
74
75
        return deferred.promise;
    }

    /**
     * Create a new directory recursively in azure file storage
     * @param fileServerClient
76
     * @param azureDirectory
SparkSnail's avatar
SparkSnail committed
77
     */
78
    export async function createDirectoryRecursive(fileServerClient: azureStorage.FileService, azureDirectory: string,
79
80
                                                   azureShare: any): Promise<boolean> {
        const deferred: Deferred<boolean> = new Deferred<boolean>();
81
82
83
        const directories: string[] = azureDirectory.split('/');
        let rootDirectory: string = '';
        for (const directory of directories) {
SparkSnail's avatar
SparkSnail committed
84
            rootDirectory += directory;
85
86
87
88
89
            let result:boolean = await createDirectory(fileServerClient, rootDirectory, azureShare);
            if (!result) {
                deferred.resolve(false);
                return deferred.promise;
            }
SparkSnail's avatar
SparkSnail committed
90
91
            rootDirectory += '/';
        }
92
        deferred.resolve(true);
93

SparkSnail's avatar
SparkSnail committed
94
95
        return deferred.promise;
    }
96

SparkSnail's avatar
SparkSnail committed
97
98
    /**
     * upload a file to azure storage
99
100
101
102
103
     * @param fileServerClient
     * @param azureDirectory
     * @param azureFileName
     * @param azureShare
     * @param localFilePath
SparkSnail's avatar
SparkSnail committed
104
     */
105
    async function uploadFileToAzure(fileServerClient: any, azureDirectory: string, azureFileName: any, azureShare: any,
106
107
                                     localFilePath: string): Promise<boolean> {
        const deferred: Deferred<boolean> = new Deferred<boolean>();
108
109
110
111
112
        await fileServerClient.createFileFromLocalFile(azureShare, azureDirectory, azureFileName, localFilePath,
                                                       (error: any, result: any, response: any) => {
            if (error) {
                getLogger()
                  .error(`Upload file failed:, ${error}`);
113
                deferred.resolve(false);
114
            } else {
115
                deferred.resolve(true);
SparkSnail's avatar
SparkSnail committed
116
            }
117
118
        });

SparkSnail's avatar
SparkSnail committed
119
120
        return deferred.promise;
    }
121

SparkSnail's avatar
SparkSnail committed
122
123
    /**
     * download a file from azure storage
124
125
126
127
128
     * @param fileServerClient
     * @param azureDirectory
     * @param azureFileName
     * @param azureShare
     * @param localFilePath
SparkSnail's avatar
SparkSnail committed
129
     */
130
    async function downloadFile(fileServerClient: any, azureDirectory: string, azureFileName: any, azureShare: any,
131
132
                                localFilePath: string): Promise<boolean> {
        const deferred: Deferred<boolean> = new Deferred<boolean>();
133
134
135
136
137
138
        // tslint:disable-next-line:non-literal-fs-path
        await fileServerClient.getFileToStream(azureShare, azureDirectory, azureFileName, fs.createWriteStream(localFilePath),
                                               (error: any, result: any, response: any) => {
            if (error) {
                getLogger()
                  .error(`Download file failed:, ${error}`);
139
                deferred.resolve(false);
140
            } else {
141
                deferred.resolve(true);
SparkSnail's avatar
SparkSnail committed
142
            }
143
144
        });

SparkSnail's avatar
SparkSnail committed
145
146
147
148
149
150
151
152
153
154
        return deferred.promise;
    }

    /**
     * Upload a directory to azure file storage
     * @param fileServerClient : the client of file server
     * @param azureDirectory : the directory in azure file storage
     * @param azureShare : the azure share used
     * @param localDirectory : local directory to be uploaded
     */
155
156
    // tslint:disable:non-literal-fs-path
    export async function uploadDirectory(fileServerClient: azureStorage.FileService, azureDirectory: string, azureShare: any,
157
158
                                          localDirectory: string): Promise<boolean> {
        const deferred: Deferred<boolean> = new Deferred<boolean>();
SparkSnail's avatar
SparkSnail committed
159
        const fileNameArray: string[] = fs.readdirSync(localDirectory);
160
161
162
163
164
        let result: boolean = await createDirectoryRecursive(fileServerClient, azureDirectory, azureShare);
        if (!result) {
            deferred.resolve(false);
            return deferred.promise;
        }
165
        for (const fileName of fileNameArray) {
SparkSnail's avatar
SparkSnail committed
166
167
            const fullFilePath: string = path.join(localDirectory, fileName);
            try {
168
169
                let resultUploadFile: boolean = true;
                let resultUploadDir: boolean = true;
170
171
                if (fs.lstatSync(fullFilePath)
                      .isFile()) {
172
                    resultUploadFile = await uploadFileToAzure(fileServerClient, azureDirectory, fileName, azureShare, fullFilePath);
SparkSnail's avatar
SparkSnail committed
173
174
                } else {
                    // If filePath is a directory, recuisively copy it to azure
175
176
177
178
179
                    resultUploadDir = await uploadDirectory(fileServerClient, String.Format('{0}/{1}', azureDirectory, fileName), azureShare, fullFilePath);
                }
                if (!(resultUploadFile && resultUploadDir)) {
                    deferred.resolve(false);
                    return deferred.promise;
SparkSnail's avatar
SparkSnail committed
180
                }
181
            } catch (error) {
182
                deferred.resolve(false);
183

SparkSnail's avatar
SparkSnail committed
184
185
186
187
                return deferred.promise;
            }
        }
        // All files/directories are copied successfully, resolve
188
        deferred.resolve(true);
189

SparkSnail's avatar
SparkSnail committed
190
191
        return deferred.promise;
    }
192

SparkSnail's avatar
SparkSnail committed
193
194
    /**
     * downlod a directory from azure
195
196
197
198
     * @param fileServerClient
     * @param azureDirectory
     * @param azureShare
     * @param localDirectory
SparkSnail's avatar
SparkSnail committed
199
     */
200
201
    export async function downloadDirectory(fileServerClient: any, azureDirectory: string, azureShare: any, localDirectory: string):
     Promise<void> {
SparkSnail's avatar
SparkSnail committed
202
        const deferred: Deferred<void> = new Deferred<void>();
203
204
205
206
207
208
        await mkDirP(localDirectory);
        fileServerClient.listFilesAndDirectoriesSegmented(azureShare, azureDirectory, 'null',
                                                          async (error: any, result: any, response: any) => {
            if (('entries' in result) === false) {
                getLogger()
                  .error(`list files failed, can't get entries in result`);
SparkSnail's avatar
SparkSnail committed
209
210
211
                throw new Error(`list files failed, can't get entries in result`);
            }

212
213
214
            if (('files' in result.entries) === false) {
                getLogger()
                  .error(`list files failed, can't get files in result['entries']`);
SparkSnail's avatar
SparkSnail committed
215
216
217
                throw new Error(`list files failed, can't get files in result['entries']`);
            }

218
219
220
            if (('directories' in result.directories) === false) {
                getLogger()
                  .error(`list files failed, can't get directories in result['entries']`);
SparkSnail's avatar
SparkSnail committed
221
222
223
                throw new Error(`list files failed, can't get directories in result['entries']`);
            }

224
            for (const fileName of result.entries.files) {
SparkSnail's avatar
SparkSnail committed
225
                const fullFilePath: string = path.join(localDirectory, fileName.name);
226
                await downloadFile(fileServerClient, azureDirectory, fileName.name, azureShare, fullFilePath);
SparkSnail's avatar
SparkSnail committed
227
            }
228

229
230
231
232
            for (const directoryName of result.entries.directories) {
                const fullDirectoryPath: string = path.join(localDirectory, directoryName.name);
                const fullAzureDirectory: string = path.join(azureDirectory, directoryName.name);
                await downloadDirectory(fileServerClient, fullAzureDirectory, azureShare, fullDirectoryPath);
SparkSnail's avatar
SparkSnail committed
233
234
            }
            deferred.resolve();
235
236
        });

SparkSnail's avatar
SparkSnail committed
237
238
239
        return deferred.promise;
    }
}
240
// tslint:enable: no-redundant-jsdoc no-any no-unsafe-any