hdfsClientUtility.test.ts

/**
 * Copyright (c) Microsoft Corporation
 * All rights reserved.
 *
 * MIT License
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
 * documentation files (the "Software"), to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
 * to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
 * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

'use strict';
import * as chai from 'chai';
import * as chaiAsPromised from 'chai-as-promised';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import * as tmp from 'tmp';
import { cleanupUnitTest, prepareUnitTest, uniqueString } from '../../common/utils';
import { HDFSClientUtility } from '../pai/hdfsClientUtility';

var WebHDFS = require('webhdfs');
var rmdir = require('rmdir');

describe('WebHDFS', function () {
    /*
    To enable web HDFS client unit test, HDFS information needs to be configured in:
    Default/.vscode/hdfsInfo.json,  whose content looks like:
    {
        "user": "user1",
        "port": 50070,
        "host": "10.0.0.0"        
    }
    */
    let skip: boolean = false;
    let testHDFSInfo: any;
    let hdfsClient: any;
    try {
        testHDFSInfo = JSON.parse(fs.readFileSync('../../.vscode/hdfsInfo.json', 'utf8'));
        console.log(testHDFSInfo);        
        hdfsClient = WebHDFS.createClient({
            user: testHDFSInfo.user,
            port: testHDFSInfo.port,
            host: testHDFSInfo.host
        });
    } catch (err) {
        console.log('Please configure rminfo.json to enable remote machine unit test.');
        skip = true;
    }

    before(() => {
        chai.should();
        chai.use(chaiAsPromised);
        tmp.setGracefulCleanup();
        prepareUnitTest();
    });

    after(() => {
        cleanupUnitTest();
    });

    it('Test HDFS utility path functions', async () => {
        if (skip) {
            return;
        }
        const testPath : string = '/nni_unittest_' + uniqueString(6);
        let exists : boolean = await HDFSClientUtility.pathExists(testPath, hdfsClient);
        // The new random named path is expected to not exist
        chai.expect(exists).to.be.equals(false);

        const mkdirResult : boolean = await HDFSClientUtility.mkdir(testPath, hdfsClient);
        // Mkdir is expected to be successful
        chai.expect(mkdirResult).to.be.equals(true);

        exists = await HDFSClientUtility.pathExists(testPath, hdfsClient);
        // The newly created path is expected to exist
        chai.expect(exists).to.be.equals(true);

        const deleteResult : boolean = await HDFSClientUtility.deletePath(testPath, hdfsClient);
        // Delete path is expected to be successful
        chai.expect(deleteResult).to.be.equals(true);

        exists = await HDFSClientUtility.pathExists(testPath, hdfsClient);
        // The deleted path is not expected to exist
        chai.expect(exists).to.be.equals(false);
    });

    it('Test HDFS utility copyFileToHdfs', async() => {
        if (skip) {
            return;
        }
        // Prepare local directory and files
        const tmpLocalDirectoryPath : string = path.join(os.tmpdir(), 'nni_unittest_dir_' + uniqueString(6));
        const tmpDataFilePath : string = path.join(tmpLocalDirectoryPath, 'file_' + uniqueString(6));
        const testFileData : string = 'TestContent123';
        fs.mkdirSync(tmpLocalDirectoryPath);
        fs.writeFileSync(tmpDataFilePath, testFileData);

        const testHDFSFilePath : string = '/nni_unittest_' + uniqueString(6);
        let exists : boolean = await HDFSClientUtility.pathExists(testHDFSFilePath, hdfsClient);
        // The new random named path is expected to not exist
        chai.expect(exists).to.be.equals(false);

        await HDFSClientUtility.copyFileToHdfs(tmpDataFilePath, testHDFSFilePath, hdfsClient);
        exists = await HDFSClientUtility.pathExists(testHDFSFilePath, hdfsClient);
        // After copy local file to HDFS, the target file path in HDFS is expected to exist
        chai.expect(exists).to.be.equals(true);

        const buffer : Buffer = await HDFSClientUtility.readFileFromHDFS(testHDFSFilePath, hdfsClient);
        const actualFileData : string = buffer.toString('utf8');
        // The file content read from HDFS is expected to equal to the content of local file
        chai.expect(actualFileData).to.be.equals(testFileData);

        const testHDFSDirPath : string = path.join('/nni_unittest_' + uniqueString(6) +  '_dir');
        
        await HDFSClientUtility.copyDirectoryToHdfs(tmpLocalDirectoryPath, testHDFSDirPath, hdfsClient);

        const files : any[] = await HDFSClientUtility.readdir(testHDFSDirPath, hdfsClient);

        // Expected file count under HDFS target directory is 1
        chai.expect(files.length).to.be.equals(1);

        // Expected file name under HDFS target directory is equal to local file name
        chai.expect(files[0].pathSuffix).to.be.equals(path.parse(tmpDataFilePath).base);

        // Cleanup
        rmdir(tmpLocalDirectoryPath);
        
        let deleteRestult : boolean = await HDFSClientUtility.deletePath(testHDFSFilePath, hdfsClient);
        chai.expect(deleteRestult).to.be.equals(true);

        deleteRestult = await HDFSClientUtility.deletePath(testHDFSDirPath, hdfsClient);
        chai.expect(deleteRestult).to.be.equals(true);
    });
});