Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
OpenDAS
nni
Commits
75028bd7
Unverified
Commit
75028bd7
authored
Mar 17, 2020
by
SparkSnail
Committed by
GitHub
Mar 17, 2020
Browse files
Merge pull request #235 from microsoft/master
merge master
parents
1d74ae5e
2e42d1d8
Changes
94
Hide whitespace changes
Inline
Side-by-side
Showing
14 changed files
with
249 additions
and
42 deletions
+249
-42
src/webui/src/components/trial-detail/TableList.tsx
src/webui/src/components/trial-detail/TableList.tsx
+29
-19
src/webui/src/static/function.ts
src/webui/src/static/function.ts
+16
-4
src/webui/src/static/model/trial.ts
src/webui/src/static/model/trial.ts
+5
-4
src/webui/src/static/style/overview.scss
src/webui/src/static/style/overview.scss
+8
-0
test/config_test.py
test/config_test.py
+1
-1
test/generate_ts_config.py
test/generate_ts_config.py
+22
-1
test/pipelines-it-pai.yml
test/pipelines-it-pai.yml
+3
-3
test/pipelines-it-paiYarn.yml
test/pipelines-it-paiYarn.yml
+59
-0
test/training_service.yml
test/training_service.yml
+16
-1
tools/nni_cmd/config_schema.py
tools/nni_cmd/config_schema.py
+25
-1
tools/nni_cmd/launcher.py
tools/nni_cmd/launcher.py
+40
-3
tools/nni_cmd/launcher_utils.py
tools/nni_cmd/launcher_utils.py
+8
-4
tools/nni_cmd/nnictl.py
tools/nni_cmd/nnictl.py
+5
-1
tools/nni_cmd/nnictl_utils.py
tools/nni_cmd/nnictl_utils.py
+12
-0
No files found.
src/webui/src/components/trial-detail/TableList.tsx
View file @
75028bd7
...
...
@@ -31,7 +31,6 @@ echarts.registerTheme('my_theme', {
color
:
'
#3c8dbc
'
});
interface
TableListProps
{
pageSize
:
number
;
tableSource
:
Array
<
TableRecord
>
;
...
...
@@ -142,7 +141,7 @@ class TableList extends React.Component<TableListProps, TableListState> {
key
:
'
sequenceId
'
,
fieldName
:
'
sequenceId
'
,
minWidth
:
80
,
maxWidth
:
1
20
,
maxWidth
:
2
4
0
,
className
:
'
tableHead
'
,
data
:
'
string
'
,
onColumnClick
:
this
.
onColumnClick
,
...
...
@@ -166,7 +165,7 @@ class TableList extends React.Component<TableListProps, TableListState> {
key
:
'
startTime
'
,
fieldName
:
'
startTime
'
,
minWidth
:
150
,
maxWidth
:
2
00
,
maxWidth
:
4
00
,
isResizable
:
true
,
data
:
'
number
'
,
onColumnClick
:
this
.
onColumnClick
,
...
...
@@ -179,8 +178,8 @@ class TableList extends React.Component<TableListProps, TableListState> {
name
:
'
End Time
'
,
key
:
'
endTime
'
,
fieldName
:
'
endTime
'
,
minWidth
:
15
0
,
maxWidth
:
2
00
,
minWidth
:
20
0
,
maxWidth
:
4
00
,
isResizable
:
true
,
data
:
'
number
'
,
onColumnClick
:
this
.
onColumnClick
,
...
...
@@ -194,7 +193,7 @@ class TableList extends React.Component<TableListProps, TableListState> {
key
:
'
duration
'
,
fieldName
:
'
duration
'
,
minWidth
:
150
,
maxWidth
:
2
00
,
maxWidth
:
3
00
,
isResizable
:
true
,
data
:
'
number
'
,
onColumnClick
:
this
.
onColumnClick
,
...
...
@@ -209,7 +208,7 @@ class TableList extends React.Component<TableListProps, TableListState> {
fieldName
:
'
status
'
,
className
:
'
tableStatus
'
,
minWidth
:
150
,
maxWidth
:
2
0
0
,
maxWidth
:
2
5
0
,
isResizable
:
true
,
data
:
'
string
'
,
onColumnClick
:
this
.
onColumnClick
,
...
...
@@ -241,17 +240,26 @@ class TableList extends React.Component<TableListProps, TableListState> {
// support intermediate result is dict because the last intermediate result is
// final result in a succeed trial, it may be a dict.
// get intermediate result dict keys array
let
otherkeys
:
string
[]
=
[
'
default
'
];
const
{
intermediateKey
}
=
this
.
state
;
const
otherkeys
:
string
[]
=
[
];
if
(
res
.
data
.
length
!==
0
)
{
otherkeys
=
Object
.
keys
(
parseMetrics
(
res
.
data
[
0
].
data
));
// just add type=number keys
const
intermediateMetrics
=
parseMetrics
(
res
.
data
[
0
].
data
);
for
(
const
key
in
intermediateMetrics
){
if
(
typeof
intermediateMetrics
[
key
]
===
'
number
'
)
{
otherkeys
.
push
(
key
);
}
}
}
// intermediateArr just store default val
Object
.
keys
(
res
.
data
).
map
(
item
=>
{
const
temp
=
parseMetrics
(
res
.
data
[
item
].
data
);
if
(
typeof
temp
===
'
object
'
)
{
intermediateArr
.
push
(
temp
.
default
);
}
else
{
intermediateArr
.
push
(
temp
);
if
(
res
.
data
[
item
].
type
===
'
PERIODICAL
'
)
{
const
temp
=
parseMetrics
(
res
.
data
[
item
].
data
);
if
(
typeof
temp
===
'
object
'
)
{
intermediateArr
.
push
(
temp
[
intermediateKey
]);
}
else
{
intermediateArr
.
push
(
temp
);
}
}
});
const
intermediate
=
intermediateGraphOption
(
intermediateArr
,
id
);
...
...
@@ -276,11 +284,13 @@ class TableList extends React.Component<TableListProps, TableListState> {
// just watch default key-val
if
(
isShowDefault
===
true
)
{
Object
.
keys
(
intermediateData
).
map
(
item
=>
{
const
temp
=
parseMetrics
(
intermediateData
[
item
].
data
);
if
(
typeof
temp
===
'
object
'
)
{
intermediateArr
.
push
(
temp
[
value
]);
}
else
{
intermediateArr
.
push
(
temp
);
if
(
intermediateData
[
item
].
type
===
'
PERIODICAL
'
)
{
const
temp
=
parseMetrics
(
intermediateData
[
item
].
data
);
if
(
typeof
temp
===
'
object
'
)
{
intermediateArr
.
push
(
temp
[
value
]);
}
else
{
intermediateArr
.
push
(
temp
);
}
}
});
}
else
{
...
...
src/webui/src/static/function.ts
View file @
75028bd7
...
...
@@ -45,6 +45,10 @@ function parseMetrics(metricData: string): any {
}
}
const
isArrayType
=
(
list
:
any
):
boolean
|
undefined
=>
{
return
Array
.
isArray
(
list
);
}
// get final result value
// draw Accuracy point graph
const
getFinalResult
=
(
final
?:
MetricDataRecord
[]):
number
=>
{
...
...
@@ -52,12 +56,14 @@ const getFinalResult = (final?: MetricDataRecord[]): number => {
let
showDefault
=
0
;
if
(
final
)
{
acc
=
parseMetrics
(
final
[
final
.
length
-
1
].
data
);
if
(
typeof
(
acc
)
===
'
object
'
)
{
if
(
typeof
(
acc
)
===
'
object
'
&&
!
isArrayType
(
acc
)
)
{
if
(
acc
.
default
)
{
showDefault
=
acc
.
default
;
}
}
else
{
}
else
if
(
typeof
(
acc
)
===
'
number
'
)
{
showDefault
=
acc
;
}
else
{
showDefault
=
NaN
;
}
return
showDefault
;
}
else
{
...
...
@@ -72,8 +78,13 @@ const getFinal = (final?: MetricDataRecord[]): FinalType | undefined => {
showDefault
=
parseMetrics
(
final
[
final
.
length
-
1
].
data
);
if
(
typeof
showDefault
===
'
number
'
)
{
showDefault
=
{
default
:
showDefault
};
return
showDefault
;
}
else
if
(
isArrayType
(
showDefault
))
{
// not support final type
return
undefined
;
}
else
if
(
typeof
showDefault
===
'
object
'
&&
showDefault
.
hasOwnProperty
(
'
default
'
))
{
return
showDefault
;
}
return
showDefault
;
}
else
{
return
undefined
;
}
...
...
@@ -205,5 +216,6 @@ function formatAccuracy(accuracy: number): string {
export
{
convertTime
,
convertDuration
,
getFinalResult
,
getFinal
,
downFile
,
intermediateGraphOption
,
killJob
,
filterByStatus
,
filterDuration
,
formatAccuracy
,
formatTimestamp
,
metricAccuracy
,
parseMetrics
formatAccuracy
,
formatTimestamp
,
metricAccuracy
,
parseMetrics
,
isArrayType
};
src/webui/src/static/model/trial.ts
View file @
75028bd7
import
{
MetricDataRecord
,
TrialJobInfo
,
TableObj
,
TableRecord
,
Parameters
,
FinalType
}
from
'
../interface
'
;
import
{
getFinal
,
formatAccuracy
,
metricAccuracy
,
parseMetrics
}
from
'
../function
'
;
import
{
getFinal
,
formatAccuracy
,
metricAccuracy
,
parseMetrics
,
isArrayType
}
from
'
../function
'
;
class
Trial
implements
TableObj
{
private
metricsInitialized
:
boolean
=
false
;
...
...
@@ -55,9 +55,11 @@ class Trial implements TableObj {
}
else
if
(
this
.
intermediates
.
length
>
0
)
{
const
temp
=
this
.
intermediates
[
this
.
intermediates
.
length
-
1
];
if
(
temp
!==
undefined
)
{
if
(
typeof
parseMetrics
(
temp
.
data
)
===
'
object
'
)
{
if
(
isArrayType
(
parseMetrics
(
temp
.
data
)))
{
return
undefined
;
}
else
if
(
typeof
parseMetrics
(
temp
.
data
)
===
'
object
'
&&
parseMetrics
(
temp
.
data
).
hasOwnProperty
(
'
default
'
))
{
return
parseMetrics
(
temp
.
data
).
default
;
}
else
{
}
else
if
(
typeof
parseMetrics
(
temp
.
data
)
===
'
number
'
)
{
return
parseMetrics
(
temp
.
data
);
}
}
else
{
...
...
@@ -67,7 +69,6 @@ class Trial implements TableObj {
return
undefined
;
}
}
/* table obj start */
get
tableRecord
():
TableRecord
{
...
...
src/webui/src/static/style/overview.scss
View file @
75028bd7
...
...
@@ -56,4 +56,12 @@
color
:
#333
;
font-size
:
14px
;
}
}
/* overview-succeed-graph */
.showMess
{
position
:
absolute
;
top
:
50%
;
left
:
50%
;
transform
:
translate
(
-50%
,
-50%
);
}
\ No newline at end of file
test/config_test.py
View file @
75028bd7
...
...
@@ -112,7 +112,7 @@ if __name__ == '__main__':
parser
=
argparse
.
ArgumentParser
()
parser
.
add_argument
(
"--config"
,
type
=
str
,
default
=
None
)
parser
.
add_argument
(
"--exclude"
,
type
=
str
,
default
=
None
)
parser
.
add_argument
(
"--ts"
,
type
=
str
,
choices
=
[
'local'
,
'remote'
,
'pai'
,
'kubeflow'
,
'frameworkcontroller'
],
default
=
'local'
)
parser
.
add_argument
(
"--ts"
,
type
=
str
,
choices
=
[
'local'
,
'remote'
,
'pai'
,
'paiYarn'
,
'kubeflow'
,
'frameworkcontroller'
],
default
=
'local'
)
parser
.
add_argument
(
"--local_gpu"
,
action
=
'store_true'
)
parser
.
add_argument
(
"--preinstall"
,
action
=
'store_true'
)
args
=
parser
.
parse_args
()
...
...
test/generate_ts_config.py
View file @
75028bd7
...
...
@@ -12,7 +12,7 @@ def update_training_service_config(args):
config
=
get_yml_content
(
TRAINING_SERVICE_FILE
)
if
args
.
nni_manager_ip
is
not
None
:
config
[
args
.
ts
][
'nniManagerIp'
]
=
args
.
nni_manager_ip
if
args
.
ts
==
'pai'
:
if
args
.
ts
==
'pai
Yarn
'
:
if
args
.
pai_user
is
not
None
:
config
[
args
.
ts
][
'paiYarnConfig'
][
'userName'
]
=
args
.
pai_user
if
args
.
pai_pwd
is
not
None
:
...
...
@@ -27,6 +27,23 @@ def update_training_service_config(args):
config
[
args
.
ts
][
'trial'
][
'outputDir'
]
=
args
.
output_dir
if
args
.
vc
is
not
None
:
config
[
args
.
ts
][
'trial'
][
'virtualCluster'
]
=
args
.
vc
if
args
.
ts
==
'pai'
:
if
args
.
pai_user
is
not
None
:
config
[
args
.
ts
][
'paiConfig'
][
'userName'
]
=
args
.
pai_user
if
args
.
pai_host
is
not
None
:
config
[
args
.
ts
][
'paiConfig'
][
'host'
]
=
args
.
pai_host
if
args
.
pai_token
is
not
None
:
config
[
args
.
ts
][
'paiConfig'
][
'token'
]
=
args
.
pai_token
if
args
.
nni_docker_image
is
not
None
:
config
[
args
.
ts
][
'trial'
][
'image'
]
=
args
.
nni_docker_image
if
args
.
nni_manager_nfs_mount_path
is
not
None
:
config
[
args
.
ts
][
'trial'
][
'nniManagerNFSMountPath'
]
=
args
.
nni_manager_nfs_mount_path
if
args
.
container_nfs_mount_path
is
not
None
:
config
[
args
.
ts
][
'trial'
][
'containerNFSMountPath'
]
=
args
.
container_nfs_mount_path
if
args
.
pai_storage_plugin
is
not
None
:
config
[
args
.
ts
][
'trial'
][
'paiStoragePlugin'
]
=
args
.
pai_storage_plugin
if
args
.
vc
is
not
None
:
config
[
args
.
ts
][
'trial'
][
'virtualCluster'
]
=
args
.
vc
elif
args
.
ts
==
'kubeflow'
:
if
args
.
nfs_server
is
not
None
:
config
[
args
.
ts
][
'kubeflowConfig'
][
'nfs'
][
'server'
]
=
args
.
nfs_server
...
...
@@ -94,6 +111,10 @@ if __name__ == '__main__':
parser
.
add_argument
(
"--data_dir"
,
type
=
str
)
parser
.
add_argument
(
"--output_dir"
,
type
=
str
)
parser
.
add_argument
(
"--vc"
,
type
=
str
)
parser
.
add_argument
(
"--pai_token"
,
type
=
str
)
parser
.
add_argument
(
"--pai_storage_plugin"
,
type
=
str
)
parser
.
add_argument
(
"--nni_manager_nfs_mount_path"
,
type
=
str
)
parser
.
add_argument
(
"--container_nfs_mount_path"
,
type
=
str
)
# args for kubeflow and frameworkController
parser
.
add_argument
(
"--nfs_server"
,
type
=
str
)
parser
.
add_argument
(
"--nfs_path"
,
type
=
str
)
...
...
test/pipelines-it-pai.yml
View file @
75028bd7
...
...
@@ -51,9 +51,9 @@ jobs:
echo "TEST_IMG:$TEST_IMG"
cd test
python3 generate_ts_config.py --ts pai --pai_host $(pai_host) --pai_user $(pai_user) --
pai_pwd $(pai_pwd) --vc $(pai_virtual_cluster)
\
--
nni_docker_image $TEST_IMG --data_dir $(data_dir) --output_dir $(output_dir
) --nni_manager_ip $(nni_manager_ip)
python3 generate_ts_config.py --ts pai --pai_host $(pai_host) --pai_user $(pai_user) --
nni_docker_image $TEST_IMG --pai_storage_plugin $(pai_storage_plugin)
\
--
pai_token $(pai_token) --nni_manager_nfs_mount_path $(nni_manager_nfs_mount_path) --container_nfs_mount_path $(container_nfs_mount_path
) --nni_manager_ip $(nni_manager_ip)
PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts pai
PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts pai
--exclude multi_phase
PATH=$HOME/.local/bin:$PATH python3 metrics_test.py
displayName
:
'
integration
test'
test/pipelines-it-paiYarn.yml
0 → 100644
View file @
75028bd7
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.
jobs
:
-
job
:
'
integration_test_paiYarn'
timeoutInMinutes
:
0
steps
:
-
script
:
python3 -m pip install --upgrade pip setuptools --user
displayName
:
'
Install
python
tools'
-
script
:
|
cd deployment/pypi
echo 'building prerelease package...'
make build
ls $(Build.SourcesDirectory)/deployment/pypi/dist/
condition
:
eq( variables['build_docker_img'], 'true' )
displayName
:
'
build
nni
bdsit_wheel'
-
script
:
|
source install.sh
displayName
:
'
Install
nni
toolkit
via
source
code'
-
script
:
|
sudo apt-get install swig -y
PATH=$HOME/.local/bin:$PATH nnictl package install --name=SMAC
PATH=$HOME/.local/bin:$PATH nnictl package install --name=BOHB
displayName
:
'
Install
dependencies
for
integration
tests
in
PAI
mode'
-
script
:
|
set -e
if [ $(build_docker_img) = 'true' ]
then
cd deployment/pypi
docker login -u $(docker_hub_user) -p $(docker_hub_pwd)
echo 'updating docker file for installing nni from local...'
# update Dockerfile to install NNI in docker image from whl file built in last step
sed -ie 's/RUN python3 -m pip --no-cache-dir install nni/COPY .\/dist\/* .\nRUN python3 -m pip install nni-*.whl/' ../docker/Dockerfile
cat ../docker/Dockerfile
export IMG_TAG=`date -u +%y%m%d%H%M`
echo 'build and upload docker image'
docker build -f ../docker/Dockerfile -t $(test_docker_img_name):$IMG_TAG .
docker push $(test_docker_img_name):$IMG_TAG
export TEST_IMG=$(test_docker_img_name):$IMG_TAG
cd ../../
else
export TEST_IMG=$(existing_docker_img)
fi
echo "TEST_IMG:$TEST_IMG"
cd test
python3 generate_ts_config.py --ts paiYarn --pai_host $(pai_host) --pai_user $(pai_user) --pai_pwd $(pai_pwd) --vc $(pai_virtual_cluster) \
--nni_docker_image $TEST_IMG --data_dir $(data_dir) --output_dir $(output_dir) --nni_manager_ip $(nni_manager_ip)
PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts paiYarn
PATH=$HOME/.local/bin:$PATH python3 metrics_test.py
displayName
:
'
integration
test'
test/training_service.yml
View file @
75028bd7
...
...
@@ -52,7 +52,7 @@ frameworkcontroller:
local
:
trainingServicePlatform
:
local
pai
:
pai
Yarn
:
nniManagerIp
:
maxExecDuration
:
15m
paiYarnConfig
:
...
...
@@ -68,6 +68,21 @@ pai:
memoryMB
:
8192
outputDir
:
virtualCluster
:
pai
:
nniManagerIp
:
maxExecDuration
:
15m
paiConfig
:
host
:
userName
:
trainingServicePlatform
:
pai
trial
:
gpuNum
:
1
cpuNum
:
1
image
:
memoryMB
:
8192
nniManagerNFSMountPath
:
containerNFSMountPath
:
paiStoragePlugin
:
remote
:
machineList
:
-
ip
:
...
...
tools/nni_cmd/config_schema.py
View file @
75028bd7
...
...
@@ -32,7 +32,8 @@ common_schema = {
'trialConcurrency'
:
setNumberRange
(
'trialConcurrency'
,
int
,
1
,
99999
),
Optional
(
'maxExecDuration'
):
And
(
Regex
(
r
'^[1-9][0-9]*[s|m|h|d]$'
,
error
=
'ERROR: maxExecDuration format is [digit]{s,m,h,d}'
)),
Optional
(
'maxTrialNum'
):
setNumberRange
(
'maxTrialNum'
,
int
,
1
,
99999
),
'trainingServicePlatform'
:
setChoice
(
'trainingServicePlatform'
,
'remote'
,
'local'
,
'pai'
,
'kubeflow'
,
'frameworkcontroller'
,
'paiYarn'
),
'trainingServicePlatform'
:
setChoice
(
'trainingServicePlatform'
,
'remote'
,
'local'
,
'pai'
,
'kubeflow'
,
'frameworkcontroller'
,
'paiYarn'
,
'dlts'
),
Optional
(
'searchSpacePath'
):
And
(
os
.
path
.
exists
,
error
=
SCHEMA_PATH_ERROR
%
'searchSpacePath'
),
Optional
(
'multiPhase'
):
setType
(
'multiPhase'
,
bool
),
Optional
(
'multiThread'
):
setType
(
'multiThread'
,
bool
),
...
...
@@ -297,6 +298,27 @@ pai_config_schema = {
})
}
dlts_trial_schema
=
{
'trial'
:{
'command'
:
setType
(
'command'
,
str
),
'codeDir'
:
setPathCheck
(
'codeDir'
),
'gpuNum'
:
setNumberRange
(
'gpuNum'
,
int
,
0
,
99999
),
'image'
:
setType
(
'image'
,
str
),
}
}
dlts_config_schema
=
{
'dltsConfig'
:
{
'dashboard'
:
setType
(
'dashboard'
,
str
),
Optional
(
'cluster'
):
setType
(
'cluster'
,
str
),
Optional
(
'team'
):
setType
(
'team'
,
str
),
Optional
(
'email'
):
setType
(
'email'
,
str
),
Optional
(
'password'
):
setType
(
'password'
,
str
),
}
}
kubeflow_trial_schema
=
{
'trial'
:{
'codeDir'
:
setPathCheck
(
'codeDir'
),
...
...
@@ -438,6 +460,8 @@ PAI_CONFIG_SCHEMA = Schema({**common_schema, **pai_trial_schema, **pai_config_sc
PAI_YARN_CONFIG_SCHEMA
=
Schema
({
**
common_schema
,
**
pai_yarn_trial_schema
,
**
pai_yarn_config_schema
})
DLTS_CONFIG_SCHEMA
=
Schema
({
**
common_schema
,
**
dlts_trial_schema
,
**
dlts_config_schema
})
KUBEFLOW_CONFIG_SCHEMA
=
Schema
({
**
common_schema
,
**
kubeflow_trial_schema
,
**
kubeflow_config_schema
})
FRAMEWORKCONTROLLER_CONFIG_SCHEMA
=
Schema
({
**
common_schema
,
**
frameworkcontroller_trial_schema
,
**
frameworkcontroller_config_schema
})
tools/nni_cmd/launcher.py
View file @
75028bd7
...
...
@@ -99,7 +99,7 @@ def start_rest_server(port, platform, mode, config_file_name, foreground=False,
node_command
=
'node'
if
sys
.
platform
==
'win32'
:
node_command
=
os
.
path
.
join
(
entry_dir
[:
-
3
],
'Scripts'
,
'node.exe'
)
cmds
=
[
node_command
,
entry_file
,
'--port'
,
str
(
port
),
'--mode'
,
platform
]
cmds
=
[
node_command
,
'--max-old-space-size=4096'
,
entry_file
,
'--port'
,
str
(
port
),
'--mode'
,
platform
]
if
mode
==
'view'
:
cmds
+=
[
'--start_mode'
,
'resume'
]
cmds
+=
[
'--readonly'
,
'true'
]
...
...
@@ -289,6 +289,25 @@ def set_frameworkcontroller_config(experiment_config, port, config_file_name):
#set trial_config
return
set_trial_config
(
experiment_config
,
port
,
config_file_name
),
err_message
def
set_dlts_config
(
experiment_config
,
port
,
config_file_name
):
'''set dlts configuration'''
dlts_config_data
=
dict
()
dlts_config_data
[
'dlts_config'
]
=
experiment_config
[
'dltsConfig'
]
response
=
rest_put
(
cluster_metadata_url
(
port
),
json
.
dumps
(
dlts_config_data
),
REST_TIME_OUT
)
err_message
=
None
if
not
response
or
not
response
.
status_code
==
200
:
if
response
is
not
None
:
err_message
=
response
.
text
_
,
stderr_full_path
=
get_log_path
(
config_file_name
)
with
open
(
stderr_full_path
,
'a+'
)
as
fout
:
fout
.
write
(
json
.
dumps
(
json
.
loads
(
err_message
),
indent
=
4
,
sort_keys
=
True
,
separators
=
(
','
,
':'
)))
return
False
,
err_message
result
,
message
=
setNNIManagerIp
(
experiment_config
,
port
,
config_file_name
)
if
not
result
:
return
result
,
message
#set trial_config
return
set_trial_config
(
experiment_config
,
port
,
config_file_name
),
err_message
def
set_experiment
(
experiment_config
,
mode
,
port
,
config_file_name
):
'''Call startExperiment (rest POST /experiment) with yaml file content'''
request_data
=
dict
()
...
...
@@ -389,6 +408,8 @@ def set_platform_config(platform, experiment_config, port, config_file_name, res
config_result
,
err_msg
=
set_kubeflow_config
(
experiment_config
,
port
,
config_file_name
)
elif
platform
==
'frameworkcontroller'
:
config_result
,
err_msg
=
set_frameworkcontroller_config
(
experiment_config
,
port
,
config_file_name
)
elif
platform
==
'dlts'
:
config_result
,
err_msg
=
set_dlts_config
(
experiment_config
,
port
,
config_file_name
)
else
:
raise
Exception
(
ERROR_INFO
%
'Unsupported platform!'
)
exit
(
1
)
...
...
@@ -525,7 +546,15 @@ def create_experiment(args):
nni_config
.
set_config
(
'experimentConfig'
,
experiment_config
)
nni_config
.
set_config
(
'restServerPort'
,
args
.
port
)
launch_experiment
(
args
,
experiment_config
,
'new'
,
config_file_name
)
try
:
launch_experiment
(
args
,
experiment_config
,
'new'
,
config_file_name
)
except
Exception
as
exception
:
nni_config
=
Config
(
config_file_name
)
restServerPid
=
nni_config
.
get_config
(
'restServerPid'
)
if
restServerPid
:
kill_command
(
restServerPid
)
print_error
(
exception
)
exit
(
1
)
def
manage_stopped_experiment
(
args
,
mode
):
'''view a stopped experiment'''
...
...
@@ -553,8 +582,16 @@ def manage_stopped_experiment(args, mode):
new_config_file_name
=
''
.
join
(
random
.
sample
(
string
.
ascii_letters
+
string
.
digits
,
8
))
new_nni_config
=
Config
(
new_config_file_name
)
new_nni_config
.
set_config
(
'experimentConfig'
,
experiment_config
)
launch_experiment
(
args
,
experiment_config
,
mode
,
new_config_file_name
,
experiment_id
)
new_nni_config
.
set_config
(
'restServerPort'
,
args
.
port
)
try
:
launch_experiment
(
args
,
experiment_config
,
mode
,
new_config_file_name
,
experiment_id
)
except
Exception
as
exception
:
nni_config
=
Config
(
new_config_file_name
)
restServerPid
=
nni_config
.
get_config
(
'restServerPid'
)
if
restServerPid
:
kill_command
(
restServerPid
)
print_error
(
exception
)
exit
(
1
)
def
view_experiment
(
args
):
'''view a stopped experiment'''
...
...
tools/nni_cmd/launcher_utils.py
View file @
75028bd7
...
...
@@ -5,8 +5,9 @@ import os
import
json
from
schema
import
SchemaError
from
schema
import
Schema
from
.config_schema
import
LOCAL_CONFIG_SCHEMA
,
REMOTE_CONFIG_SCHEMA
,
PAI_CONFIG_SCHEMA
,
PAI_YARN_CONFIG_SCHEMA
,
KUBEFLOW_CONFIG_SCHEMA
,
\
FRAMEWORKCONTROLLER_CONFIG_SCHEMA
,
tuner_schema_dict
,
advisor_schema_dict
,
assessor_schema_dict
from
.config_schema
import
LOCAL_CONFIG_SCHEMA
,
REMOTE_CONFIG_SCHEMA
,
PAI_CONFIG_SCHEMA
,
PAI_YARN_CONFIG_SCHEMA
,
\
DLTS_CONFIG_SCHEMA
,
KUBEFLOW_CONFIG_SCHEMA
,
FRAMEWORKCONTROLLER_CONFIG_SCHEMA
,
\
tuner_schema_dict
,
advisor_schema_dict
,
assessor_schema_dict
from
.common_utils
import
print_error
,
print_warning
,
print_normal
,
get_yml_content
def
expand_path
(
experiment_config
,
key
):
...
...
@@ -147,7 +148,9 @@ def validate_kubeflow_operators(experiment_config):
def
validate_common_content
(
experiment_config
):
'''Validate whether the common values in experiment_config is valid'''
if
not
experiment_config
.
get
(
'trainingServicePlatform'
)
or
\
experiment_config
.
get
(
'trainingServicePlatform'
)
not
in
[
'local'
,
'remote'
,
'pai'
,
'kubeflow'
,
'frameworkcontroller'
,
'paiYarn'
]:
experiment_config
.
get
(
'trainingServicePlatform'
)
not
in
[
'local'
,
'remote'
,
'pai'
,
'kubeflow'
,
'frameworkcontroller'
,
'paiYarn'
,
'dlts'
]:
print_error
(
'Please set correct trainingServicePlatform!'
)
exit
(
1
)
schema_dict
=
{
...
...
@@ -156,7 +159,8 @@ def validate_common_content(experiment_config):
'pai'
:
PAI_CONFIG_SCHEMA
,
'paiYarn'
:
PAI_YARN_CONFIG_SCHEMA
,
'kubeflow'
:
KUBEFLOW_CONFIG_SCHEMA
,
'frameworkcontroller'
:
FRAMEWORKCONTROLLER_CONFIG_SCHEMA
'frameworkcontroller'
:
FRAMEWORKCONTROLLER_CONFIG_SCHEMA
,
'dlts'
:
DLTS_CONFIG_SCHEMA
,
}
separate_schema_dict
=
{
'tuner'
:
tuner_schema_dict
,
...
...
tools/nni_cmd/nnictl.py
View file @
75028bd7
...
...
@@ -11,7 +11,7 @@ from .updater import update_searchspace, update_concurrency, update_duration, up
from
.nnictl_utils
import
stop_experiment
,
trial_ls
,
trial_kill
,
list_experiment
,
experiment_status
,
\
log_trial
,
experiment_clean
,
platform_clean
,
experiment_list
,
\
monitor_experiment
,
export_trials_data
,
trial_codegen
,
webui_url
,
\
get_config
,
log_stdout
,
log_stderr
,
search_space_auto_gen
get_config
,
log_stdout
,
log_stderr
,
search_space_auto_gen
,
webui_nas
from
.package_management
import
package_install
,
package_show
from
.constants
import
DEFAULT_REST_PORT
from
.tensorboard_utils
import
start_tensorboard
,
stop_tensorboard
...
...
@@ -158,6 +158,10 @@ def parse_args():
parser_webui_url
=
parser_webui_subparsers
.
add_parser
(
'url'
,
help
=
'show the url of web ui'
)
parser_webui_url
.
add_argument
(
'id'
,
nargs
=
'?'
,
help
=
'the id of experiment'
)
parser_webui_url
.
set_defaults
(
func
=
webui_url
)
parser_webui_nas
=
parser_webui_subparsers
.
add_parser
(
'nas'
,
help
=
'show nas ui'
)
parser_webui_nas
.
add_argument
(
'--port'
,
default
=
6060
,
type
=
int
,
help
=
'port of nas ui'
)
parser_webui_nas
.
add_argument
(
'--logdir'
,
default
=
'.'
,
type
=
str
,
help
=
'the logdir where nas ui will read data'
)
parser_webui_nas
.
set_defaults
(
func
=
webui_nas
)
#parse config command
parser_config
=
subparsers
.
add_parser
(
'config'
,
help
=
'get config information'
)
...
...
tools/nni_cmd/nnictl_utils.py
View file @
75028bd7
...
...
@@ -8,6 +8,7 @@ import json
import
time
import
re
import
shutil
import
subprocess
from
datetime
import
datetime
,
timezone
from
pathlib
import
Path
from
subprocess
import
Popen
...
...
@@ -388,6 +389,17 @@ def webui_url(args):
nni_config
=
Config
(
get_config_filename
(
args
))
print_normal
(
'{0} {1}'
.
format
(
'Web UI url:'
,
' '
.
join
(
nni_config
.
get_config
(
'webuiUrl'
))))
def
webui_nas
(
args
):
'''launch nas ui'''
print_normal
(
'Starting NAS UI...'
)
# TODO: find file path on installing with pypi
# TODO: use correct node on win32
try
:
cmds
=
[
'node'
,
'src/nasui/server.js'
,
'--port'
,
str
(
args
.
port
),
'--logdir'
,
args
.
logdir
]
subprocess
.
run
(
cmds
)
except
KeyboardInterrupt
:
pass
def
local_clean
(
directory
):
'''clean up local data'''
print_normal
(
'removing folder {0}'
.
format
(
directory
))
...
...
Prev
1
2
3
4
5
Next
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment