Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
OpenDAS
nni
Commits
715b1899
Unverified
Commit
715b1899
authored
Feb 25, 2021
by
SparkSnail
Committed by
GitHub
Feb 25, 2021
Browse files
Fix reuse mode env folder (#3374)
parent
0d37a64a
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
5 additions
and
7 deletions
+5
-7
ts/nni_manager/training_service/reusable/environments/amlEnvironmentService.ts
...ng_service/reusable/environments/amlEnvironmentService.ts
+1
-1
ts/nni_manager/training_service/reusable/environments/localEnvironmentService.ts
..._service/reusable/environments/localEnvironmentService.ts
+2
-4
ts/nni_manager/training_service/reusable/environments/remoteEnvironmentService.ts
...service/reusable/environments/remoteEnvironmentService.ts
+1
-1
ts/nni_manager/training_service/reusable/trialDispatcher.ts
ts/nni_manager/training_service/reusable/trialDispatcher.ts
+1
-1
No files found.
ts/nni_manager/training_service/reusable/environments/amlEnvironmentService.ts
View file @
715b1899
...
...
@@ -113,7 +113,7 @@ export class AMLEnvironmentService extends EnvironmentService {
throw
new
Error
(
'
AML trial config is not initialized
'
);
}
const
amlEnvironment
:
AMLEnvironmentInformation
=
environment
as
AMLEnvironmentInformation
;
const
environmentLocalTempFolder
=
path
.
join
(
this
.
experimentRootDir
,
this
.
experimentId
,
"
environment-temp
"
);
const
environmentLocalTempFolder
=
path
.
join
(
this
.
experimentRootDir
,
"
environment-temp
"
);
environment
.
command
=
`import os\nos.system('mv envs outputs/envs && cd outputs &&
${
amlEnvironment
.
command
}
')`
;
environment
.
useActiveGpu
=
this
.
amlClusterConfig
.
useActiveGpu
;
environment
.
maxTrialNumberPerGpu
=
this
.
amlClusterConfig
.
maxTrialNumPerGpu
;
...
...
ts/nni_manager/training_service/reusable/environments/localEnvironmentService.ts
View file @
715b1899
...
...
@@ -101,7 +101,7 @@ export class LocalEnvironmentService extends EnvironmentService {
`Write $LASTEXITCODE " " $NOW_DATE | Out-File "
${
path
.
join
(
environment
.
runnerWorkingFolder
,
'
code
'
)}
" -NoNewline -encoding utf8`
);
}
else
{
script
.
push
(
`cd
${
this
.
experimentRootDir
}
`
);
script
.
push
(
`eval
${
environment
.
command
}
--job_pid_file
${
environment
.
runnerWorkingFolder
}
/pid 1>
${
environment
.
runnerWorkingFolder
}
/trialrunner_stdout 2>
${
environment
.
runnerWorkingFolder
}
/trialrunner_stderr
"
`
);
script
.
push
(
`eval
${
environment
.
command
}
--job_pid_file
${
environment
.
runnerWorkingFolder
}
/pid 1>
${
environment
.
runnerWorkingFolder
}
/trialrunner_stdout 2>
${
environment
.
runnerWorkingFolder
}
/trialrunner_stderr`
);
if
(
process
.
platform
===
'
darwin
'
)
{
// https://superuser.com/questions/599072/how-to-get-bash-execution-time-in-milliseconds-under-mac-os-x
// Considering the worst case, write 999 to avoid negative duration
...
...
@@ -118,9 +118,7 @@ export class LocalEnvironmentService extends EnvironmentService {
if
(
this
.
localTrialConfig
===
undefined
)
{
throw
new
Error
(
'
Local trial config is not initialized
'
);
}
// Need refactor, this temp folder path is not appropriate, there are two expId in this path
const
localTempFolder
:
string
=
path
.
join
(
this
.
experimentRootDir
,
this
.
experimentId
,
"
environment-temp
"
,
"
envs
"
);
const
localTempFolder
:
string
=
path
.
join
(
this
.
experimentRootDir
,
"
environment-temp
"
,
"
envs
"
);
const
localEnvCodeFolder
:
string
=
path
.
join
(
this
.
experimentRootDir
,
"
envs
"
);
environment
.
runnerWorkingFolder
=
path
.
join
(
localEnvCodeFolder
,
environment
.
id
);
await
execMkdir
(
environment
.
runnerWorkingFolder
);
...
...
ts/nni_manager/training_service/reusable/environments/remoteEnvironmentService.ts
View file @
715b1899
...
...
@@ -264,7 +264,7 @@ ${environment.command} --job_pid_file ${environment.runnerWorkingFolder}/pid \
}
const
executor
=
await
this
.
getExecutor
(
environment
.
id
);
const
environmentLocalTempFolder
:
string
=
path
.
join
(
this
.
experimentRootDir
,
this
.
experimentId
,
"
environment-temp
"
)
path
.
join
(
this
.
experimentRootDir
,
"
environment-temp
"
)
await
executor
.
createFolder
(
environment
.
runnerWorkingFolder
);
await
execMkdir
(
environmentLocalTempFolder
);
await
fs
.
promises
.
writeFile
(
path
.
join
(
environmentLocalTempFolder
,
executor
.
getScriptName
(
"
run
"
)),
...
...
ts/nni_manager/training_service/reusable/trialDispatcher.ts
View file @
715b1899
...
...
@@ -201,7 +201,7 @@ class TrialDispatcher implements TrainingService {
}
else
{
this
.
log
.
debug
(
`TrialDispatcher: create temp storage service to temp folder.`
);
storageService
=
new
MountedStorageService
();
const
environmentLocalTempFolder
=
path
.
join
(
this
.
experimentRootDir
,
this
.
experimentId
,
"
environment-temp
"
);
const
environmentLocalTempFolder
=
path
.
join
(
this
.
experimentRootDir
,
"
environment-temp
"
);
storageService
.
initialize
(
this
.
trialConfig
.
codeDir
,
environmentLocalTempFolder
);
}
// Copy the compressed file to remoteDirectory and delete it
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment