Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
OpenDAS
nni
Commits
715b1899
"...resnet50_tensorflow.git" did not exist on "5633969b4680b94df697729f5f1c2d14bd7feabc"
Unverified
Commit
715b1899
authored
Feb 25, 2021
by
SparkSnail
Committed by
GitHub
Feb 25, 2021
Browse files
Fix reuse mode env folder (#3374)
parent
0d37a64a
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
5 additions
and
7 deletions
+5
-7
ts/nni_manager/training_service/reusable/environments/amlEnvironmentService.ts
...ng_service/reusable/environments/amlEnvironmentService.ts
+1
-1
ts/nni_manager/training_service/reusable/environments/localEnvironmentService.ts
..._service/reusable/environments/localEnvironmentService.ts
+2
-4
ts/nni_manager/training_service/reusable/environments/remoteEnvironmentService.ts
...service/reusable/environments/remoteEnvironmentService.ts
+1
-1
ts/nni_manager/training_service/reusable/trialDispatcher.ts
ts/nni_manager/training_service/reusable/trialDispatcher.ts
+1
-1
No files found.
ts/nni_manager/training_service/reusable/environments/amlEnvironmentService.ts
View file @
715b1899
...
@@ -113,7 +113,7 @@ export class AMLEnvironmentService extends EnvironmentService {
...
@@ -113,7 +113,7 @@ export class AMLEnvironmentService extends EnvironmentService {
throw
new
Error
(
'
AML trial config is not initialized
'
);
throw
new
Error
(
'
AML trial config is not initialized
'
);
}
}
const
amlEnvironment
:
AMLEnvironmentInformation
=
environment
as
AMLEnvironmentInformation
;
const
amlEnvironment
:
AMLEnvironmentInformation
=
environment
as
AMLEnvironmentInformation
;
const
environmentLocalTempFolder
=
path
.
join
(
this
.
experimentRootDir
,
this
.
experimentId
,
"
environment-temp
"
);
const
environmentLocalTempFolder
=
path
.
join
(
this
.
experimentRootDir
,
"
environment-temp
"
);
environment
.
command
=
`import os\nos.system('mv envs outputs/envs && cd outputs &&
${
amlEnvironment
.
command
}
')`
;
environment
.
command
=
`import os\nos.system('mv envs outputs/envs && cd outputs &&
${
amlEnvironment
.
command
}
')`
;
environment
.
useActiveGpu
=
this
.
amlClusterConfig
.
useActiveGpu
;
environment
.
useActiveGpu
=
this
.
amlClusterConfig
.
useActiveGpu
;
environment
.
maxTrialNumberPerGpu
=
this
.
amlClusterConfig
.
maxTrialNumPerGpu
;
environment
.
maxTrialNumberPerGpu
=
this
.
amlClusterConfig
.
maxTrialNumPerGpu
;
...
...
ts/nni_manager/training_service/reusable/environments/localEnvironmentService.ts
View file @
715b1899
...
@@ -101,7 +101,7 @@ export class LocalEnvironmentService extends EnvironmentService {
...
@@ -101,7 +101,7 @@ export class LocalEnvironmentService extends EnvironmentService {
`Write $LASTEXITCODE " " $NOW_DATE | Out-File "
${
path
.
join
(
environment
.
runnerWorkingFolder
,
'
code
'
)}
" -NoNewline -encoding utf8`
);
`Write $LASTEXITCODE " " $NOW_DATE | Out-File "
${
path
.
join
(
environment
.
runnerWorkingFolder
,
'
code
'
)}
" -NoNewline -encoding utf8`
);
}
else
{
}
else
{
script
.
push
(
`cd
${
this
.
experimentRootDir
}
`
);
script
.
push
(
`cd
${
this
.
experimentRootDir
}
`
);
script
.
push
(
`eval
${
environment
.
command
}
--job_pid_file
${
environment
.
runnerWorkingFolder
}
/pid 1>
${
environment
.
runnerWorkingFolder
}
/trialrunner_stdout 2>
${
environment
.
runnerWorkingFolder
}
/trialrunner_stderr
"
`
);
script
.
push
(
`eval
${
environment
.
command
}
--job_pid_file
${
environment
.
runnerWorkingFolder
}
/pid 1>
${
environment
.
runnerWorkingFolder
}
/trialrunner_stdout 2>
${
environment
.
runnerWorkingFolder
}
/trialrunner_stderr`
);
if
(
process
.
platform
===
'
darwin
'
)
{
if
(
process
.
platform
===
'
darwin
'
)
{
// https://superuser.com/questions/599072/how-to-get-bash-execution-time-in-milliseconds-under-mac-os-x
// https://superuser.com/questions/599072/how-to-get-bash-execution-time-in-milliseconds-under-mac-os-x
// Considering the worst case, write 999 to avoid negative duration
// Considering the worst case, write 999 to avoid negative duration
...
@@ -118,9 +118,7 @@ export class LocalEnvironmentService extends EnvironmentService {
...
@@ -118,9 +118,7 @@ export class LocalEnvironmentService extends EnvironmentService {
if
(
this
.
localTrialConfig
===
undefined
)
{
if
(
this
.
localTrialConfig
===
undefined
)
{
throw
new
Error
(
'
Local trial config is not initialized
'
);
throw
new
Error
(
'
Local trial config is not initialized
'
);
}
}
// Need refactor, this temp folder path is not appropriate, there are two expId in this path
const
localTempFolder
:
string
=
path
.
join
(
this
.
experimentRootDir
,
"
environment-temp
"
,
"
envs
"
);
const
localTempFolder
:
string
=
path
.
join
(
this
.
experimentRootDir
,
this
.
experimentId
,
"
environment-temp
"
,
"
envs
"
);
const
localEnvCodeFolder
:
string
=
path
.
join
(
this
.
experimentRootDir
,
"
envs
"
);
const
localEnvCodeFolder
:
string
=
path
.
join
(
this
.
experimentRootDir
,
"
envs
"
);
environment
.
runnerWorkingFolder
=
path
.
join
(
localEnvCodeFolder
,
environment
.
id
);
environment
.
runnerWorkingFolder
=
path
.
join
(
localEnvCodeFolder
,
environment
.
id
);
await
execMkdir
(
environment
.
runnerWorkingFolder
);
await
execMkdir
(
environment
.
runnerWorkingFolder
);
...
...
ts/nni_manager/training_service/reusable/environments/remoteEnvironmentService.ts
View file @
715b1899
...
@@ -264,7 +264,7 @@ ${environment.command} --job_pid_file ${environment.runnerWorkingFolder}/pid \
...
@@ -264,7 +264,7 @@ ${environment.command} --job_pid_file ${environment.runnerWorkingFolder}/pid \
}
}
const
executor
=
await
this
.
getExecutor
(
environment
.
id
);
const
executor
=
await
this
.
getExecutor
(
environment
.
id
);
const
environmentLocalTempFolder
:
string
=
const
environmentLocalTempFolder
:
string
=
path
.
join
(
this
.
experimentRootDir
,
this
.
experimentId
,
"
environment-temp
"
)
path
.
join
(
this
.
experimentRootDir
,
"
environment-temp
"
)
await
executor
.
createFolder
(
environment
.
runnerWorkingFolder
);
await
executor
.
createFolder
(
environment
.
runnerWorkingFolder
);
await
execMkdir
(
environmentLocalTempFolder
);
await
execMkdir
(
environmentLocalTempFolder
);
await
fs
.
promises
.
writeFile
(
path
.
join
(
environmentLocalTempFolder
,
executor
.
getScriptName
(
"
run
"
)),
await
fs
.
promises
.
writeFile
(
path
.
join
(
environmentLocalTempFolder
,
executor
.
getScriptName
(
"
run
"
)),
...
...
ts/nni_manager/training_service/reusable/trialDispatcher.ts
View file @
715b1899
...
@@ -201,7 +201,7 @@ class TrialDispatcher implements TrainingService {
...
@@ -201,7 +201,7 @@ class TrialDispatcher implements TrainingService {
}
else
{
}
else
{
this
.
log
.
debug
(
`TrialDispatcher: create temp storage service to temp folder.`
);
this
.
log
.
debug
(
`TrialDispatcher: create temp storage service to temp folder.`
);
storageService
=
new
MountedStorageService
();
storageService
=
new
MountedStorageService
();
const
environmentLocalTempFolder
=
path
.
join
(
this
.
experimentRootDir
,
this
.
experimentId
,
"
environment-temp
"
);
const
environmentLocalTempFolder
=
path
.
join
(
this
.
experimentRootDir
,
"
environment-temp
"
);
storageService
.
initialize
(
this
.
trialConfig
.
codeDir
,
environmentLocalTempFolder
);
storageService
.
initialize
(
this
.
trialConfig
.
codeDir
,
environmentLocalTempFolder
);
}
}
// Copy the compressed file to remoteDirectory and delete it
// Copy the compressed file to remoteDirectory and delete it
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment