Add BOM to support non-English characters in PowerShell scripts (#5164)

ts/nni_manager/common/shellUtils.ts

 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
+import fsPromises from 'fs/promises';
+
 // for readability
 const singleQuote = "'";
 const doubleQuote = '"';
@@ -59,3 +61,13 @@ export function powershellString(str: string): string {
         return singleQuote + str + singleQuote;
     }
 }
+
+export function createScriptFile(path: string, content: string): Promise<void> {
+    // eslint-disable-next-line no-control-regex
+    if (path.endsWith('.ps1') && !/^[\x00-\x7F]*$/.test(content)) {
+        // PowerShell does not use UTF-8 by default.
+        // Add BOM to inform it if the script contains non-ASCII characters.
+        content = '\uFEFF' + content;
+    }
+    return fsPromises.writeFile(path, content, { mode: 0o777 });
+}

ts/nni_manager/test/common/shell_utils.test.ts

+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+import assert from 'assert/strict';
+import fs from 'fs';
+import fsPromises from 'fs/promises';
+import os from 'os';
+import path from 'path';
+
+import { createScriptFile } from 'common/shellUtils';
+
+let tempDir: string | null = null;
+const asciiScript = 'echo hello\necho world';
+const unicodeScript = 'echo 你好\necho 世界';
+
+/* test cases */
+
+async function testAscii(): Promise<void> {
+    const file = path.join(tempDir!, 'test1.ps1');
+    await createScriptFile(file, asciiScript);
+    const script = await fsPromises.readFile(file, { encoding: 'utf8' });
+    assert.equal(script, asciiScript);
+}
+
+async function testUnicode(): Promise<void> {
+    const file = path.join(tempDir!, 'test2.ps1');
+    await createScriptFile(file, unicodeScript);
+    const script = await fsPromises.readFile(file, { encoding: 'utf8' });
+    assert.equal(script, '\uFEFF' + unicodeScript);
+}
+
+async function testBash(): Promise<void> {
+    const file = path.join(tempDir!, 'test.sh');
+    await createScriptFile(file, unicodeScript);
+    const script = await fsPromises.readFile(file, { encoding: 'utf8' });
+    assert.equal(script, unicodeScript);
+}
+
+/* environment */
+
+function beforeHook() {
+    tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'nni-ut-'));
+}
+
+function afterHook() {
+    if (tempDir) {
+        fs.rmSync(tempDir, { force: true, recursive: true });
+    }
+}
+
+/* register */
+
+describe('## common.shell_utils ##', () => {
+    before(beforeHook);
+
+    it('powershell ascii', () => testAscii());
+    it('powershell unicode', () => testUnicode());
+    it('bash unicode', () => testBash());
+
+    after(afterHook);
+});

ts/nni_manager/training_service/local/localTrainingService.ts

@@ -10,7 +10,7 @@ import tkill from 'tree-kill';
 import { NNIError, NNIErrorNames } from 'common/errors';
 import { getExperimentId } from 'common/experimentStartupInfo';
 import { getLogger, Logger } from 'common/log';
-import { powershellString, shellString } from 'common/shellUtils';
+import { powershellString, shellString, createScriptFile } from 'common/shellUtils';
 import {
     HyperParameters, TrainingService, TrialJobApplicationForm,
     TrialJobDetail, TrialJobMetric, TrialJobStatus
@@ -458,8 +458,8 @@ class LocalTrainingService implements TrainingService {
         await execMkdir(path.join(trialJobDetail.workingDirectory, '.nni'));
         await execNewFile(path.join(trialJobDetail.workingDirectory, '.nni', 'metrics'));
         const scriptName: string = getScriptName('run');
-        await fs.promises.writeFile(path.join(trialJobDetail.workingDirectory, scriptName),
-                                    runScriptContent.join(getNewLine()), { encoding: 'utf8', mode: 0o777 });
+        await createScriptFile(path.join(trialJobDetail.workingDirectory, scriptName),
+                runScriptContent.join(getNewLine()));
         await this.writeParameterFile(trialJobDetail.workingDirectory, trialJobDetail.form.hyperParameters);
         const trialJobProcess: cp.ChildProcess = runScript(path.join(trialJobDetail.workingDirectory, scriptName));
         this.setTrialJobStatus(trialJobDetail, 'RUNNING');

ts/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts

@@ -29,6 +29,7 @@ import {
     ExecutorManager, RemoteMachineScheduleInfo, RemoteMachineScheduleResult, RemoteMachineTrialJobDetail
 } from './remoteMachineData';
 import { RemoteMachineJobRestServer } from './remoteMachineJobRestServer';
+import { createScriptFile } from 'common/shellUtils';
 
 /**
  * Training Service implementation for Remote Machine (Linux)
@@ -510,9 +511,9 @@ class RemoteMachineTrainingService implements TrainingService {
         await execMkdir(path.join(trialLocalTempFolder, '.nni'));
 
         // Write install_nni.sh, it's not used in Windows platform.
-        await fs.promises.writeFile(path.join(trialLocalTempFolder, executor.getScriptName("install_nni")), CONTAINER_INSTALL_NNI_SHELL_FORMAT, { encoding: 'utf8' });
+        await createScriptFile(path.join(trialLocalTempFolder, executor.getScriptName("install_nni")), CONTAINER_INSTALL_NNI_SHELL_FORMAT);
         // Write file content ( run.sh and parameter.cfg ) to local tmp files
-        await fs.promises.writeFile(path.join(trialLocalTempFolder, executor.getScriptName("run")), runScriptTrialContent, { encoding: 'utf8' });
+        await createScriptFile(path.join(trialLocalTempFolder, executor.getScriptName("run")), runScriptTrialContent);
         await this.writeParameterFile(trialJobId, form.hyperParameters);
         // Copy files in codeDir to remote working directory
         await executor.copyDirectoryToRemote(trialLocalTempFolder, trialJobDetail.workingDirectory);

ts/nni_manager/training_service/reusable/environments/localEnvironmentService.ts

@@ -8,7 +8,7 @@ import * as component from 'common/component';
 import { getLogger, Logger } from 'common/log';
 import { ExperimentConfig } from 'common/experimentConfig';
 import { ExperimentStartupInfo } from 'common/experimentStartupInfo';
-import { powershellString } from 'common/shellUtils';
+import { powershellString, createScriptFile } from 'common/shellUtils';
 import { EnvironmentInformation, EnvironmentService } from '../environment';
 import { isAlive, getNewLine } from 'common/utils';
 import { execMkdir, runScript, getScriptName, execCopydir } from 'training_service/common/util';
@@ -121,8 +121,7 @@ export class LocalEnvironmentService extends EnvironmentService {
         await execMkdir(environment.runnerWorkingFolder);
         environment.command = this.getScript(environment).join(getNewLine());
         const scriptName: string = getScriptName('run');
-        await fs.promises.writeFile(path.join(localEnvCodeFolder, scriptName),
-                                    environment.command, { encoding: 'utf8', mode: 0o777 });
+        await createScriptFile(path.join(localEnvCodeFolder, scriptName), environment.command);
 
         // Execute command in local machine
         runScript(path.join(localEnvCodeFolder, scriptName));

ts/nni_manager/training_service/reusable/environments/remoteEnvironmentService.ts

@@ -13,7 +13,8 @@ import { execMkdir } from 'training_service/common/util';
 import { ExecutorManager } from 'training_service/remote_machine/remoteMachineData';
 import { ShellExecutor } from 'training_service/remote_machine/shellExecutor';
 import { RemoteMachineEnvironmentInformation } from '../remote/remoteConfig';
-import { SharedStorageService } from '../sharedStorage'
+import { SharedStorageService } from '../sharedStorage';
+import { createScriptFile } from 'common/shellUtils';
 
 @component.Singleton
 export class RemoteEnvironmentService extends EnvironmentService {
@@ -255,8 +256,8 @@ export class RemoteEnvironmentService extends EnvironmentService {
             path.join(this.experimentRootDir, "environment-temp")
         await executor.createFolder(environment.runnerWorkingFolder);
         await execMkdir(environmentLocalTempFolder);
-        await fs.promises.writeFile(path.join(environmentLocalTempFolder, executor.getScriptName("run")),
-        environment.command, { encoding: 'utf8' });
+        await createScriptFile(path.join(environmentLocalTempFolder, executor.getScriptName("run")),
+                environment.command);
         // Copy files in codeDir to remote working directory
         await executor.copyDirectoryToRemote(environmentLocalTempFolder, this.remoteExperimentRootDir);
         // Execute command in remote machine, set isInteractive=true to run script in conda environment