Unverified Commit 4b5a1eb6 authored by chicm-ms's avatar chicm-ms Committed by GitHub
Browse files

Set pai/kubeflow integration test timeout to infinite (#778)

Set pai/kubeflow integration test job timeout to infinite.
parent 9c5c52b9
...@@ -38,7 +38,6 @@ def gen_new_config(config_file, training_service='local'): ...@@ -38,7 +38,6 @@ def gen_new_config(config_file, training_service='local'):
new_config_file = config_file + '.tmp' new_config_file = config_file + '.tmp'
ts = get_yml_content('training_service.yml')[training_service] ts = get_yml_content('training_service.yml')[training_service]
print(config)
print(ts) print(ts)
# hack for kubeflow trial config # hack for kubeflow trial config
...@@ -64,7 +63,6 @@ def run_test(config_file, training_service, local_gpu=False): ...@@ -64,7 +63,6 @@ def run_test(config_file, training_service, local_gpu=False):
return return
try: try:
print('Testing %s...' % config_file)
proc = subprocess.run(['nnictl', 'create', '--config', new_config_file]) proc = subprocess.run(['nnictl', 'create', '--config', new_config_file])
assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode
...@@ -109,8 +107,10 @@ def run(args): ...@@ -109,8 +107,10 @@ def run(args):
try: try:
# sleep 5 seconds here, to make sure previous stopped exp has enough time to exit to avoid port conflict # sleep 5 seconds here, to make sure previous stopped exp has enough time to exit to avoid port conflict
time.sleep(5) time.sleep(5)
print(GREEN + 'Testing:' + config_file + CLEAR)
begin_time = time.time()
run_test(config_file, args.ts, args.local_gpu) run_test(config_file, args.ts, args.local_gpu)
print(GREEN + 'Test %s: TEST PASS' % (config_file) + CLEAR) print(GREEN + 'Test %s: TEST PASS IN %d mins' % (config_file, (time.time() - begin_time)/60) + CLEAR)
except Exception as error: except Exception as error:
print(RED + 'Test %s: TEST FAIL' % (config_file) + CLEAR) print(RED + 'Test %s: TEST FAIL' % (config_file) + CLEAR)
print('%r' % error) print('%r' % error)
......
jobs: jobs:
- job: 'integration_test_kubeflow' - job: 'integration_test_kubeflow'
timeoutInMinutes: 0
pool: 'NNI CI KUBE CLI' pool: 'NNI CI KUBE CLI'
variables: variables:
......
jobs: jobs:
- job: 'integration_test_pai' - job: 'integration_test_pai'
timeoutInMinutes: 0
pool: 'NNI CI PAI CLI' pool: 'NNI CI PAI CLI'
variables: variables:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment